

'''simple docstring'''


import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch
						from torch import nn

						from transformers import (
						    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
						    VideoMAEForPreTraining,
						    VideoMAEForVideoClassification,
						    VideoMAEModel,
						)
						from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
						from transformers import VideoMAEImageProcessor



class 					lowercase						:


							def __init__(			self    ,			_snake_case    ,			_snake_case=13    ,			_snake_case=10    ,			_snake_case=3    ,			_snake_case=2    ,			_snake_case=2    ,			_snake_case=2    ,			_snake_case=True    ,			_snake_case=True    ,			_snake_case=32    ,			_snake_case=5    ,			_snake_case=4    ,			_snake_case=37    ,			_snake_case="gelu"    ,			_snake_case=0.1    ,			_snake_case=0.1    ,			_snake_case=10    ,			_snake_case=0.02    ,			_snake_case=0.9    ,			_snake_case=None    ,			)							->				Optional[Any]:
								UpperCAmelCase_		:      int       	=					parent
								UpperCAmelCase_		:      List[str]       	=					batch_size
								UpperCAmelCase_		:      Optional[Any]       	=					image_size
								UpperCAmelCase_		:      Union[str, Any]       	=					num_channels
								UpperCAmelCase_		:      Optional[int]       	=					patch_size
								UpperCAmelCase_		:      Optional[int]       	=					tubelet_size
								UpperCAmelCase_		:      Union[str, Any]       	=					num_frames
								UpperCAmelCase_		:      List[Any]       	=					is_training
								UpperCAmelCase_		:      List[str]       	=					use_labels
								UpperCAmelCase_		:      Optional[int]       	=					hidden_size
								UpperCAmelCase_		:      List[str]       	=					num_hidden_layers
								UpperCAmelCase_		:      Tuple       	=					num_attention_heads
								UpperCAmelCase_		:      Optional[Any]       	=					intermediate_size
								UpperCAmelCase_		:      Union[str, Any]       	=					hidden_act
								UpperCAmelCase_		:      Dict       	=					hidden_dropout_prob
								UpperCAmelCase_		:      int       	=					attention_probs_dropout_prob
								UpperCAmelCase_		:      str       	=					type_sequence_label_size
								UpperCAmelCase_		:      Optional[Any]       	=					initializer_range
								UpperCAmelCase_		:      Dict       	=					mask_ratio
								UpperCAmelCase_		:      Dict       	=					scope

								# in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
								UpperCAmelCase_		:      List[str]       	=					(image_size // patch_size) ** 2
								UpperCAmelCase_		:      Tuple       	=					(num_frames // tubelet_size) * self.num_patches_per_frame

								# use this variable to define bool_masked_pos
								UpperCAmelCase_		:      Optional[int]       	=					int(mask_ratio * self.seq_length)


							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_		:      List[str]       	=					floats_tensor(
								    [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size])

								UpperCAmelCase_		:      List[str]       	=					None
								if self.use_labels:
									UpperCAmelCase_		:      Any       	=					ids_tensor([self.batch_size]    ,			self.type_sequence_label_size)

								UpperCAmelCase_		:      Dict       	=					self.get_config()

								return config, pixel_values, labels


							def 						_snake_case							(			self)							->				Optional[int]:
								return VideoMAEConfig(
								    image_size=self.image_size    ,			patch_size=self.patch_size    ,			num_channels=self.num_channels    ,			num_frames=self.num_frames    ,			tubelet_size=self.tubelet_size    ,			hidden_size=self.hidden_size    ,			num_hidden_layers=self.num_hidden_layers    ,			num_attention_heads=self.num_attention_heads    ,			intermediate_size=self.intermediate_size    ,			hidden_act=self.hidden_act    ,			hidden_dropout_prob=self.hidden_dropout_prob    ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,			is_decoder=_snake_case    ,			initializer_range=self.initializer_range    ,			)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case)							->				Dict:
								UpperCAmelCase_		:      List[Any]       	=					VideoMAEModel(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      List[str]       	=					model(_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case)							->				Union[str, Any]:
								UpperCAmelCase_		:      str       	=					VideoMAEForPreTraining(_snake_case)
								model.to(_snake_case)
								model.eval()
								# important: each video needs to have the same number of masked patches
								# hence we define a single mask, which we then repeat for each example in the batch
								UpperCAmelCase_		:      List[Any]       	=					torch.ones((self.num_masks,))
								UpperCAmelCase_		:      List[str]       	=					torch.cat([mask, torch.zeros(self.seq_length - mask.size(0))])
								UpperCAmelCase_		:      int       	=					mask.expand(self.batch_size    ,			-1).bool()

								UpperCAmelCase_		:      str       	=					model(_snake_case    ,			_snake_case)
								# model only returns predictions for masked patches
								UpperCAmelCase_		:      Union[str, Any]       	=					mask.sum().item()
								UpperCAmelCase_		:      Any       	=					3 * self.tubelet_size * self.patch_size**2
								self.parent.assertEqual(result.logits.shape    ,			(self.batch_size, num_masked_patches, decoder_num_labels))





							def 						_snake_case							(			self)							->				Dict:
								UpperCAmelCase_		:      str       	=					self.prepare_config_and_inputs()
								UpperCAmelCase_	,    UpperCAmelCase_	,    UpperCAmelCase_		:      List[str]       	=					config_and_inputs
								UpperCAmelCase_		:      Optional[Any]       	=					{'pixel_values': pixel_values}
								return config, inputs_dict



@require_torch
class 					lowercase						(  a_,					a_,					unittest.TestCase ):
							_lowerCamelCase		:      Optional[int]=   (
							    (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
							)
							_lowerCamelCase		:      List[str]=   (
							    {"feature-extraction": VideoMAEModel, "video-classification": VideoMAEForVideoClassification}
							    if is_torch_available()
							    else {}
							)

							_lowerCamelCase		:      Any=   False
							_lowerCamelCase		:      Optional[int]=   False
							_lowerCamelCase		:      str=   False
							_lowerCamelCase		:      Optional[Any]=   False


							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_		:      List[Any]       	=					VideoMAEModelTester(self)
								UpperCAmelCase_		:      Optional[int]       	=					ConfigTester(self    ,			config_class=_snake_case    ,			has_text_modality=_snake_case    ,			hidden_size=37)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case=False)							->				Optional[int]:
								UpperCAmelCase_		:      Dict       	=					copy.deepcopy(_snake_case)

								if model_class == VideoMAEForPreTraining:
									# important: each video needs to have the same number of masked patches
									# hence we define a single mask, which we then repeat for each example in the batch
									UpperCAmelCase_		:      Any       	=					torch.ones((self.model_tester.num_masks,))
									UpperCAmelCase_		:      int       	=					torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0))])
									UpperCAmelCase_		:      Dict       	=					mask.expand(self.model_tester.batch_size    ,			-1).bool()
									UpperCAmelCase_		:      List[Any]       	=					bool_masked_pos.to(_snake_case)

								if return_labels:
									if model_class in [
									    *get_values(_snake_case),
									]:
										UpperCAmelCase_		:      List[Any]       	=					torch.zeros(
										    self.model_tester.batch_size    ,			dtype=torch.long    ,			device=_snake_case)

								return inputs_dict


							def 						_snake_case							(			self)							->				Union[str, Any]:
								self.config_tester.run_common_tests()


							@unittest.skip(reason='VideoMAE does not use inputs_embeds')
							def 						_snake_case							(			self)							->				Union[str, Any]:
								pass


							def 						_snake_case							(			self)							->				Optional[int]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Any       	=					self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
									UpperCAmelCase_		:      Union[str, Any]       	=					model_class(_snake_case)
									self.assertIsInstance(model.get_input_embeddings()    ,			(nn.Module))
									UpperCAmelCase_		:      Dict       	=					model.get_output_embeddings()
									self.assertTrue(x is None or isinstance(_snake_case    ,			nn.Linear))


							def 						_snake_case							(			self)							->				Dict:
								UpperCAmelCase_	,    UpperCAmelCase_		:      List[Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
									UpperCAmelCase_		:      Union[str, Any]       	=					model_class(_snake_case)
									UpperCAmelCase_		:      Dict       	=					inspect.signature(model.forward)
									# signature.parameters is an OrderedDict => so arg_names order is deterministic
									UpperCAmelCase_		:      Optional[Any]       	=					[*signature.parameters.keys()]

									UpperCAmelCase_		:      str       	=					['pixel_values']
									self.assertListEqual(arg_names[:1]    ,			_snake_case)


							def 						_snake_case							(			self)							->				Any:
								UpperCAmelCase_		:      Tuple       	=					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_snake_case)


							def 						_snake_case							(			self)							->				List[Any]:
								UpperCAmelCase_		:      List[str]       	=					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_pretraining(*_snake_case)


							@slow
							def 						_snake_case							(			self)							->				Union[str, Any]:
								for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									UpperCAmelCase_		:      str       	=					VideoMAEModel.from_pretrained(_snake_case)
									self.assertIsNotNone(_snake_case)


							def 						_snake_case							(			self)							->				Optional[Any]:
								if not self.has_attentions:
									pass

								else:
									UpperCAmelCase_	,    UpperCAmelCase_		:      Any       	=					self.model_tester.prepare_config_and_inputs_for_common()
									UpperCAmelCase_		:      Optional[int]       	=					True

									for model_class in self.all_model_classes:
										UpperCAmelCase_		:      Union[str, Any]       	=					self.model_tester.seq_length - self.model_tester.num_masks
										UpperCAmelCase_		:      Tuple       	=					(
										    num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
										)

										UpperCAmelCase_		:      int       	=					True
										UpperCAmelCase_		:      Union[str, Any]       	=					False
										UpperCAmelCase_		:      Dict       	=					True
										UpperCAmelCase_		:      str       	=					model_class(_snake_case)
										model.to(_snake_case)
										model.eval()
										with torch.no_grad():
											UpperCAmelCase_		:      Optional[Any]       	=					model(**self._prepare_for_class(_snake_case    ,			_snake_case))
										UpperCAmelCase_		:      Union[str, Any]       	=					outputs.attentions
										self.assertEqual(len(_snake_case)    ,			self.model_tester.num_hidden_layers)

										# check that output_attentions also work using config
										del inputs_dict["output_attentions"]
										UpperCAmelCase_		:      Optional[Any]       	=					True
										UpperCAmelCase_		:      Dict       	=					model_class(_snake_case)
										model.to(_snake_case)
										model.eval()
										with torch.no_grad():
											UpperCAmelCase_		:      List[Any]       	=					model(**self._prepare_for_class(_snake_case    ,			_snake_case))
										UpperCAmelCase_		:      List[str]       	=					outputs.attentions
										self.assertEqual(len(_snake_case)    ,			self.model_tester.num_hidden_layers)

										self.assertListEqual(
										    list(attentions[0].shape[-3:])    ,			[self.model_tester.num_attention_heads, seq_len, seq_len]    ,			)
										UpperCAmelCase_		:      Optional[Any]       	=					len(_snake_case)

										# Check attention is always last and order is fine
										UpperCAmelCase_		:      Optional[int]       	=					True
										UpperCAmelCase_		:      Optional[Any]       	=					True
										UpperCAmelCase_		:      Tuple       	=					model_class(_snake_case)
										model.to(_snake_case)
										model.eval()
										with torch.no_grad():
											UpperCAmelCase_		:      Dict       	=					model(**self._prepare_for_class(_snake_case    ,			_snake_case))

										self.assertEqual(out_len + 1    ,			len(_snake_case))

										UpperCAmelCase_		:      Union[str, Any]       	=					outputs.attentions

										self.assertEqual(len(_snake_case)    ,			self.model_tester.num_hidden_layers)
										self.assertListEqual(
										    list(self_attentions[0].shape[-3:])    ,			[self.model_tester.num_attention_heads, seq_len, seq_len]    ,			)


							def 						_snake_case							(			self)							->				Tuple:
								def check_hidden_states_output(_snake_case    ,			_snake_case    ,			_snake_case):
									UpperCAmelCase_		:      List[Any]       	=					model_class(_snake_case)
									model.to(_snake_case)
									model.eval()

									with torch.no_grad():
										UpperCAmelCase_		:      Union[str, Any]       	=					model(**self._prepare_for_class(_snake_case    ,			_snake_case))

									UpperCAmelCase_		:      Tuple       	=					outputs.hidden_states
									UpperCAmelCase_		:      Tuple       	=					self.model_tester.num_hidden_layers + 1
									self.assertEqual(len(_snake_case)    ,			_snake_case)

									UpperCAmelCase_		:      List[str]       	=					self.model_tester.seq_length - self.model_tester.num_masks
									UpperCAmelCase_		:      Union[str, Any]       	=					num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

									self.assertListEqual(
									    list(hidden_states[0].shape[-2:])    ,			[seq_length, self.model_tester.hidden_size]    ,			)

								UpperCAmelCase_	,    UpperCAmelCase_		:      Tuple       	=					self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
									UpperCAmelCase_		:      int       	=					True
									check_hidden_states_output(_snake_case    ,			_snake_case    ,			_snake_case)

									# check that output_hidden_states also work using config
									del inputs_dict["output_hidden_states"]
									UpperCAmelCase_		:      Union[str, Any]       	=					True

									check_hidden_states_output(_snake_case    ,			_snake_case    ,			_snake_case)





							@unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.')
							def 						_snake_case							(			self)							->				Optional[Any]:
								pass







def 						SCREAMING_SNAKE_CASE(  )    ->    Any:
	UpperCAmelCase_		:      Any       	=					hf_hub_download(
	    repo_id='hf-internal-testing/spaghetti-video'   ,filename='eating_spaghetti.npy'   ,repo_type='dataset'  )
	UpperCAmelCase_		:      Dict       	=					np.load(UpperCamelCase  )
	return list(UpperCamelCase  )







@require_torch
@require_vision
class 					lowercase						(  unittest.TestCase ):


							@cached_property
							def 						_snake_case							(			self)							->				Dict:
								# logits were tested with a different mean and std, so we use the same here
								return (
								    VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]    ,			image_std=[0.5, 0.5, 0.5])
								    if is_vision_available()
								    else None
								)


							@slow
							def 						_snake_case							(			self)							->				Dict:
								UpperCAmelCase_		:      Any       	=					VideoMAEForVideoClassification.from_pretrained('MCG-NJU/videomae-base-finetuned-kinetics').to(
								    _snake_case)

								UpperCAmelCase_		:      Optional[Any]       	=					self.default_image_processor
								UpperCAmelCase_		:      Optional[int]       	=					prepare_video()
								UpperCAmelCase_		:      Union[str, Any]       	=					image_processor(_snake_case    ,			return_tensors='pt').to(_snake_case)

								# forward pass
								with torch.no_grad():
									UpperCAmelCase_		:      Union[str, Any]       	=					model(**_snake_case)

								# verify the logits
								UpperCAmelCase_		:      Dict       	=					torch.Size((1, 400))
								self.assertEqual(outputs.logits.shape    ,			_snake_case)

								UpperCAmelCase_		:      int       	=					torch.tensor([0.3_669, -0.0_688, -0.2_421]).to(_snake_case)

								self.assertTrue(torch.allclose(outputs.logits[0, :3]    ,			_snake_case    ,			atol=1e-4))





							@slow
							def 						_snake_case							(			self)							->				Optional[Any]:
								UpperCAmelCase_		:      List[str]       	=					VideoMAEForPreTraining.from_pretrained('MCG-NJU/videomae-base-short').to(_snake_case)

								UpperCAmelCase_		:      List[Any]       	=					self.default_image_processor
								UpperCAmelCase_		:      Union[str, Any]       	=					prepare_video()
								UpperCAmelCase_		:      List[str]       	=					image_processor(_snake_case    ,			return_tensors='pt').to(_snake_case)

								# add boolean mask, indicating which patches to mask
								UpperCAmelCase_		:      Optional[Any]       	=					hf_hub_download(repo_id='hf-internal-testing/bool-masked-pos'    ,			filename='bool_masked_pos.pt')
								UpperCAmelCase_		:      str       	=					torch.load(_snake_case)

								# forward pass
								with torch.no_grad():
									UpperCAmelCase_		:      List[str]       	=					model(**_snake_case)

								# verify the logits
								UpperCAmelCase_		:      str       	=					torch.Size([1, 1408, 1536])
								UpperCAmelCase_		:      Dict       	=					torch.tensor(
								    [[0.7_994, 0.9_612, 0.8_508], [0.7_401, 0.8_958, 0.8_302], [0.5_862, 0.7_468, 0.7_325]]    ,			device=_snake_case)
								self.assertEqual(outputs.logits.shape    ,			_snake_case)
								self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]    ,			_snake_case    ,			atol=1e-4))

								# verify the loss (`config.norm_pix_loss` = `True`)
								UpperCAmelCase_		:      Union[str, Any]       	=					torch.tensor([0.5_142]    ,			device=_snake_case)
								self.assertTrue(torch.allclose(outputs.loss    ,			_snake_case    ,			atol=1e-4))

								# verify the loss (`config.norm_pix_loss` = `False`)
								UpperCAmelCase_		:      Optional[Any]       	=					VideoMAEForPreTraining.from_pretrained('MCG-NJU/videomae-base-short'    ,			norm_pix_loss=_snake_case).to(
								    _snake_case)

								with torch.no_grad():
									UpperCAmelCase_		:      List[str]       	=					model(**_snake_case)

								UpperCAmelCase_		:      Optional[int]       	=					torch.tensor(torch.tensor([0.6_469])    ,			device=_snake_case)
								self.assertTrue(torch.allclose(outputs.loss    ,			_snake_case    ,			atol=1e-4))









'''simple docstring'''


import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class 					lowercase						:


							def __init__(			self    ,			_snake_case    ,			_snake_case=13    ,			_snake_case=7    ,			_snake_case=True    ,			_snake_case=True    ,			_snake_case=False    ,			_snake_case=True    ,			_snake_case=99    ,			_snake_case=32    ,			_snake_case=5    ,			_snake_case=4    ,			_snake_case=37    ,			_snake_case="gelu"    ,			_snake_case=0.1    ,			_snake_case=0.1    ,			_snake_case=512    ,			_snake_case=16    ,			_snake_case=2    ,			_snake_case=0.02    ,			_snake_case=3    ,			_snake_case=4    ,			_snake_case=None    ,			)							->				str:
								UpperCAmelCase_		:      Dict       	=					parent
								UpperCAmelCase_		:      Tuple       	=					batch_size
								UpperCAmelCase_		:      Tuple       	=					seq_length
								UpperCAmelCase_		:      Dict       	=					is_training
								UpperCAmelCase_		:      Tuple       	=					use_input_mask
								UpperCAmelCase_		:      List[Any]       	=					use_token_type_ids
								UpperCAmelCase_		:      Optional[int]       	=					use_labels
								UpperCAmelCase_		:      List[str]       	=					vocab_size
								UpperCAmelCase_		:      int       	=					hidden_size
								UpperCAmelCase_		:      Any       	=					num_hidden_layers
								UpperCAmelCase_		:      Optional[int]       	=					num_attention_heads
								UpperCAmelCase_		:      List[str]       	=					intermediate_size
								UpperCAmelCase_		:      Union[str, Any]       	=					hidden_act
								UpperCAmelCase_		:      Optional[int]       	=					hidden_dropout_prob
								UpperCAmelCase_		:      Any       	=					attention_probs_dropout_prob
								UpperCAmelCase_		:      Dict       	=					max_position_embeddings
								UpperCAmelCase_		:      Union[str, Any]       	=					type_vocab_size
								UpperCAmelCase_		:      int       	=					type_sequence_label_size
								UpperCAmelCase_		:      List[str]       	=					initializer_range
								UpperCAmelCase_		:      Any       	=					num_labels
								UpperCAmelCase_		:      Optional[Any]       	=					num_choices
								UpperCAmelCase_		:      str       	=					scope


							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.vocab_size)

								UpperCAmelCase_		:      Any       	=					None
								if self.use_input_mask:
									UpperCAmelCase_		:      Optional[Any]       	=					random_attention_mask([self.batch_size, self.seq_length])

								UpperCAmelCase_		:      Union[str, Any]       	=					None
								if self.use_token_type_ids:
									UpperCAmelCase_		:      Tuple       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.type_vocab_size)

								UpperCAmelCase_		:      Dict       	=					None
								UpperCAmelCase_		:      List[str]       	=					None
								UpperCAmelCase_		:      Optional[Any]       	=					None
								if self.use_labels:
									UpperCAmelCase_		:      Any       	=					ids_tensor([self.batch_size]    ,			self.type_sequence_label_size)
									UpperCAmelCase_		:      List[str]       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.num_labels)
									UpperCAmelCase_		:      Tuple       	=					ids_tensor([self.batch_size]    ,			self.num_choices)

								UpperCAmelCase_		:      Tuple       	=					self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def 						_snake_case							(			self)							->				Optional[Any]:
								return OpenLlamaConfig(
								    vocab_size=self.vocab_size    ,			hidden_size=self.hidden_size    ,			num_hidden_layers=self.num_hidden_layers    ,			num_attention_heads=self.num_attention_heads    ,			intermediate_size=self.intermediate_size    ,			hidden_act=self.hidden_act    ,			hidden_dropout_prob=self.hidden_dropout_prob    ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,			max_position_embeddings=self.max_position_embeddings    ,			type_vocab_size=self.type_vocab_size    ,			is_decoder=_snake_case    ,			initializer_range=self.initializer_range    ,			use_stable_embedding=_snake_case    ,			)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case)							->				Union[str, Any]:
								UpperCAmelCase_		:      str       	=					OpenLlamaModel(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Dict       	=					model(_snake_case    ,			attention_mask=_snake_case)
								UpperCAmelCase_		:      List[str]       	=					model(_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				int:
								UpperCAmelCase_		:      Union[str, Any]       	=					True
								UpperCAmelCase_		:      List[Any]       	=					OpenLlamaModel(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Optional[int]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			)
								UpperCAmelCase_		:      Any       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			)
								UpperCAmelCase_		:      Optional[int]       	=					model(_snake_case    ,			attention_mask=_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				Any:
								UpperCAmelCase_		:      Optional[int]       	=					OpenLlamaForCausalLM(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Optional[int]       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.parent.assertEqual(result.logits.shape    ,			(self.batch_size, self.seq_length, self.vocab_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				Union[str, Any]:
								UpperCAmelCase_		:      Any       	=					True
								UpperCAmelCase_		:      Optional[Any]       	=					True
								UpperCAmelCase_		:      int       	=					OpenLlamaForCausalLM(config=_snake_case)
								model.to(_snake_case)
								model.eval()

								# first forward pass
								UpperCAmelCase_		:      List[str]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			use_cache=_snake_case    ,			)
								UpperCAmelCase_		:      Dict       	=					outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								UpperCAmelCase_		:      str       	=					ids_tensor((self.batch_size, 3)    ,			config.vocab_size)
								UpperCAmelCase_		:      int       	=					ids_tensor((self.batch_size, 3)    ,			vocab_size=2)

								# append to next input_ids and
								UpperCAmelCase_		:      Tuple       	=					torch.cat([input_ids, next_tokens]    ,			dim=-1)
								UpperCAmelCase_		:      List[str]       	=					torch.cat([input_mask, next_mask]    ,			dim=-1)

								UpperCAmelCase_		:      Optional[Any]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			output_hidden_states=_snake_case    ,			)['hidden_states'][0]
								UpperCAmelCase_		:      List[str]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			past_key_values=_snake_case    ,			output_hidden_states=_snake_case    ,			)['hidden_states'][0]

								# select random slice
								UpperCAmelCase_		:      Optional[Any]       	=					ids_tensor((1,)    ,			output_from_past.shape[-1]).item()
								UpperCAmelCase_		:      Optional[int]       	=					output_from_no_past[:, -3:, random_slice_idx].detach()
								UpperCAmelCase_		:      Union[str, Any]       	=					output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-3))





							def 						_snake_case							(			self)							->				List[Any]:
								UpperCAmelCase_		:      Optional[Any]       	=					self.prepare_config_and_inputs()
								(
								    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    
								)		:      Union[str, Any]       	=					config_and_inputs
								UpperCAmelCase_		:      List[str]       	=					{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict



@require_torch
class 					lowercase						(  a_,					a_,					a_,					unittest.TestCase ):
							_lowerCamelCase		:      Dict=   (
							    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
							)
							_lowerCamelCase		:      str=   (OpenLlamaForCausalLM,) if is_torch_available() else ()
							_lowerCamelCase		:      Optional[int]=   (
							    {
							        "feature-extraction": OpenLlamaModel,
							        "text-classification": OpenLlamaForSequenceClassification,
							        "text-generation": OpenLlamaForCausalLM,
							        "zero-shot": OpenLlamaForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							_lowerCamelCase		:      Tuple=   False
							_lowerCamelCase		:      int=   False


							def 						_snake_case							(			self)							->				int:
								UpperCAmelCase_		:      Any       	=					OpenLlamaModelTester(self)
								UpperCAmelCase_		:      Union[str, Any]       	=					ConfigTester(self    ,			config_class=_snake_case    ,			hidden_size=37)


							def 						_snake_case							(			self)							->				Optional[Any]:
								self.config_tester.run_common_tests()


							def 						_snake_case							(			self)							->				str:
								UpperCAmelCase_		:      Tuple       	=					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_snake_case)


							def 						_snake_case							(			self)							->				Union[str, Any]:
								UpperCAmelCase_		:      str       	=					self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
									UpperCAmelCase_		:      List[Any]       	=					type
									self.model_tester.create_and_check_model(*_snake_case)


							def 						_snake_case							(			self)							->				Union[str, Any]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      List[Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      List[str]       	=					3
								UpperCAmelCase_		:      List[str]       	=					input_dict['input_ids']
								UpperCAmelCase_		:      Optional[Any]       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.model_tester.batch_size]    ,			self.model_tester.type_sequence_label_size)
								UpperCAmelCase_		:      List[Any]       	=					OpenLlamaForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Tuple       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							def 						_snake_case							(			self)							->				Any:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Dict       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      Optional[Any]       	=					3
								UpperCAmelCase_		:      int       	=					'single_label_classification'
								UpperCAmelCase_		:      int       	=					input_dict['input_ids']
								UpperCAmelCase_		:      Any       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.model_tester.batch_size]    ,			self.model_tester.type_sequence_label_size)
								UpperCAmelCase_		:      int       	=					OpenLlamaForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      List[str]       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							def 						_snake_case							(			self)							->				Optional[int]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      List[Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      int       	=					3
								UpperCAmelCase_		:      Any       	=					'multi_label_classification'
								UpperCAmelCase_		:      Tuple       	=					input_dict['input_ids']
								UpperCAmelCase_		:      str       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Any       	=					ids_tensor(
								    [self.model_tester.batch_size, config.num_labels]    ,			self.model_tester.type_sequence_label_size).to(torch.float)
								UpperCAmelCase_		:      Optional[Any]       	=					OpenLlamaForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      List[Any]       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test')
							def 						_snake_case							(			self)							->				Optional[int]:
								pass





							@parameterized.expand([('linear',), ('dynamic',)])
							def 						_snake_case							(			self    ,			_snake_case)							->				List[str]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Any       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      List[Any]       	=					ids_tensor([1, 10]    ,			config.vocab_size)
								UpperCAmelCase_		:      List[str]       	=					ids_tensor([1, int(config.max_position_embeddings * 1.5)]    ,			config.vocab_size)

								set_seed(42)  # Fixed seed at init time so the two models get the same random weights
								UpperCAmelCase_		:      int       	=					OpenLlamaModel(_snake_case)
								original_model.to(_snake_case)
								original_model.eval()
								UpperCAmelCase_		:      Optional[Any]       	=					original_model(_snake_case).last_hidden_state
								UpperCAmelCase_		:      Tuple       	=					original_model(_snake_case).last_hidden_state

								set_seed(42)  # Fixed seed at init time so the two models get the same random weights
								UpperCAmelCase_		:      Dict       	=					{'type': scaling_type, 'factor': 10.0}
								UpperCAmelCase_		:      Union[str, Any]       	=					OpenLlamaModel(_snake_case)
								scaled_model.to(_snake_case)
								scaled_model.eval()
								UpperCAmelCase_		:      Dict       	=					scaled_model(_snake_case).last_hidden_state
								UpperCAmelCase_		:      Optional[int]       	=					scaled_model(_snake_case).last_hidden_state

								# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
								# maximum sequence length, so the outputs for the short input should match.
								if scaling_type == "dynamic":
									self.assertTrue(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-5))
								else:
									self.assertFalse(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-5))

								# The output should be different for long inputs
								self.assertFalse(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-5))






