
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel




class 				_lowerCamelCase :





     """simple docstring"""







     def __init__(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE=1_3						,						__SCREAMING_SNAKE_CASE=7						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=False						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=9_9						,						__SCREAMING_SNAKE_CASE=3_2						,						__SCREAMING_SNAKE_CASE=5						,						__SCREAMING_SNAKE_CASE=4						,						__SCREAMING_SNAKE_CASE=3_7						,						__SCREAMING_SNAKE_CASE="gelu"						,						__SCREAMING_SNAKE_CASE=0.1						,						__SCREAMING_SNAKE_CASE=0.1						,						__SCREAMING_SNAKE_CASE=5_1_2						,						__SCREAMING_SNAKE_CASE=1_6						,						__SCREAMING_SNAKE_CASE=2						,						__SCREAMING_SNAKE_CASE=0.02						,						__SCREAMING_SNAKE_CASE=3						,						__SCREAMING_SNAKE_CASE=4						,						__SCREAMING_SNAKE_CASE=None						,						)  ->   List[str]:




           """simple docstring"""







           UpperCamelCase__    :   int					=				parent
           UpperCamelCase__    :   int					=				batch_size
           UpperCamelCase__    :   Union[str, Any]					=				seq_length
           UpperCamelCase__    :   List[Any]					=				is_training
           UpperCamelCase__    :   str					=				use_input_mask
           UpperCamelCase__    :   str					=				use_token_type_ids
           UpperCamelCase__    :   str					=				use_labels
           UpperCamelCase__    :   Optional[Any]					=				vocab_size
           UpperCamelCase__    :   List[str]					=				hidden_size
           UpperCamelCase__    :   Tuple					=				num_hidden_layers
           UpperCamelCase__    :   str					=				num_attention_heads
           UpperCamelCase__    :   Union[str, Any]					=				intermediate_size
           UpperCamelCase__    :   Any					=				hidden_act
           UpperCamelCase__    :   List[str]					=				hidden_dropout_prob
           UpperCamelCase__    :   Optional[int]					=				attention_probs_dropout_prob
           UpperCamelCase__    :   int					=				max_position_embeddings
           UpperCamelCase__    :   Dict					=				type_vocab_size
           UpperCamelCase__    :   int					=				type_sequence_label_size
           UpperCamelCase__    :   str					=				initializer_range
           UpperCamelCase__    :   Dict					=				num_labels
           UpperCamelCase__    :   Dict					=				num_choices
           UpperCamelCase__    :   int					=				scope






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           UpperCamelCase__    :   List[str]					=				ids_tensor([self.batch_size, self.seq_length]						,						self.vocab_size  )

           UpperCamelCase__    :   Any					=				None
           if self.use_input_mask:
                 UpperCamelCase__    :   List[Any]					=				random_attention_mask([self.batch_size, self.seq_length]  )

           UpperCamelCase__    :   Optional[int]					=				None
           if self.use_token_type_ids:
                 UpperCamelCase__    :   List[Any]					=				ids_tensor([self.batch_size, self.seq_length]						,						self.type_vocab_size  )

           UpperCamelCase__    :   Tuple					=				None
           UpperCamelCase__    :   Optional[int]					=				None
           UpperCamelCase__    :   Optional[int]					=				None
           if self.use_labels:
                 UpperCamelCase__    :   Optional[int]					=				ids_tensor([self.batch_size]						,						self.type_sequence_label_size  )
                 UpperCamelCase__    :   Optional[int]					=				ids_tensor([self.batch_size, self.seq_length]						,						self.num_labels  )
                 UpperCamelCase__    :   List[str]					=				ids_tensor([self.batch_size]						,						self.num_choices  )

           UpperCamelCase__    :   Dict					=				self.get_config()

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           return OpenLlamaConfig(
               vocab_size=self.vocab_size						,						hidden_size=self.hidden_size						,						num_hidden_layers=self.num_hidden_layers						,						num_attention_heads=self.num_attention_heads						,						intermediate_size=self.intermediate_size						,						hidden_act=self.hidden_act						,						hidden_dropout_prob=self.hidden_dropout_prob						,						attention_probs_dropout_prob=self.attention_probs_dropout_prob						,						max_position_embeddings=self.max_position_embeddings						,						type_vocab_size=self.type_vocab_size						,						is_decoder=a_						,						initializer_range=self.initializer_range						,						use_stable_embedding=a_						,						)






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   int					=				OpenLlamaModel(config=a_  )
           model.to(a_  )
           model.eval()
           UpperCamelCase__    :   int					=				model(a_						,						attention_mask=a_  )
           UpperCamelCase__    :   Tuple					=				model(a_  )
           self.parent.assertEqual(result.last_hidden_state.shape						,						(self.batch_size, self.seq_length, self.hidden_size)  )






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						)  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				True
           UpperCamelCase__    :   str					=				OpenLlamaModel(a_  )
           model.to(a_  )
           model.eval()
           UpperCamelCase__    :   Any					=				model(
               a_						,						attention_mask=a_						,						encoder_hidden_states=a_						,						encoder_attention_mask=a_						,						)
           UpperCamelCase__    :   str					=				model(
               a_						,						attention_mask=a_						,						encoder_hidden_states=a_						,						)
           UpperCamelCase__    :   int					=				model(a_						,						attention_mask=a_  )
           self.parent.assertEqual(result.last_hidden_state.shape						,						(self.batch_size, self.seq_length, self.hidden_size)  )






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						)  ->   int:




           """simple docstring"""







           UpperCamelCase__    :   Tuple					=				OpenLlamaForCausalLM(config=a_  )
           model.to(a_  )
           model.eval()
           UpperCamelCase__    :   Optional[int]					=				model(a_						,						attention_mask=a_						,						labels=a_  )
           self.parent.assertEqual(result.logits.shape						,						(self.batch_size, self.seq_length, self.vocab_size)  )






     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE						,						)  ->   int:




           """simple docstring"""







           UpperCamelCase__    :   int					=				True
           UpperCamelCase__    :   Union[str, Any]					=				True
           UpperCamelCase__    :   List[Any]					=				OpenLlamaForCausalLM(config=a_  )
           model.to(a_  )
           model.eval()

           # first forward pass
           UpperCamelCase__    :   Union[str, Any]					=				model(
               a_						,						attention_mask=a_						,						encoder_hidden_states=a_						,						encoder_attention_mask=a_						,						use_cache=a_						,						)
           UpperCamelCase__    :   Union[str, Any]					=				outputs.past_key_values

           # create hypothetical multiple next token and extent to next_input_ids
           UpperCamelCase__    :   str					=				ids_tensor((self.batch_size, 3)						,						config.vocab_size  )
           UpperCamelCase__    :   int					=				ids_tensor((self.batch_size, 3)						,						vocab_size=2  )

           # append to next input_ids and
           UpperCamelCase__    :   str					=				torch.cat([input_ids, next_tokens]						,						dim=-1  )
           UpperCamelCase__    :   Optional[int]					=				torch.cat([input_mask, next_mask]						,						dim=-1  )

           UpperCamelCase__    :   Any					=				model(
               a_						,						attention_mask=a_						,						encoder_hidden_states=a_						,						encoder_attention_mask=a_						,						output_hidden_states=a_						,						)["hidden_states"][0]
           UpperCamelCase__    :   Union[str, Any]					=				model(
               a_						,						attention_mask=a_						,						encoder_hidden_states=a_						,						encoder_attention_mask=a_						,						past_key_values=a_						,						output_hidden_states=a_						,						)["hidden_states"][0]

           # select random slice
           UpperCamelCase__    :   Optional[Any]					=				ids_tensor((1,)						,						output_from_past.shape[-1]  ).item()
           UpperCamelCase__    :   int					=				output_from_no_past[:, -3:, random_slice_idx].detach()
           UpperCamelCase__    :   Tuple					=				output_from_past[:, :, random_slice_idx].detach()

           self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

           # test that outputs are equal for slice
           self.parent.assertTrue(torch.allclose(a_						,						a_						,						atol=1e-3  )  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Optional[int]:




           """simple docstring"""







           UpperCamelCase__    :   str					=				self.prepare_config_and_inputs()
           (
               UpperCamelCase__
           )    :   Optional[Any]					=				config_and_inputs
           UpperCamelCase__    :   List[str]					=				{"input_ids": input_ids, "attention_mask": input_mask}
           return config, inputs_dict




@require_torch
class 				_lowerCamelCase (      __lowerCamelCase	,							__lowerCamelCase	,							__lowerCamelCase	,							unittest.TestCase ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  (
         (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
     )
     SCREAMING_SNAKE_CASE_		   =  (OpenLlamaForCausalLM,) if is_torch_available() else ()
     SCREAMING_SNAKE_CASE_		   =  (
         {
             '''feature-extraction''': OpenLlamaModel,
             '''text-classification''': OpenLlamaForSequenceClassification,
             '''text-generation''': OpenLlamaForCausalLM,
             '''zero-shot''': OpenLlamaForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     SCREAMING_SNAKE_CASE_		   =  False
     SCREAMING_SNAKE_CASE_		   =  False






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Optional[Any]:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				OpenLlamaModelTester(self  )
           UpperCamelCase__    :   List[Any]					=				ConfigTester(self						,						config_class=a_						,						hidden_size=3_7  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           self.config_tester.run_common_tests()






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           UpperCamelCase__    :   str					=				self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*a_  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           UpperCamelCase__    :   int					=				self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
                 UpperCamelCase__    :   Union[str, Any]					=				type
                 self.model_tester.create_and_check_model(*a_  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Union[str, Any]:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				self.model_tester.prepare_config_and_inputs_for_common()
           UpperCamelCase__    :   Optional[int]					=				3
           UpperCamelCase__    :   Dict					=				input_dict["input_ids"]
           UpperCamelCase__    :   str					=				input_ids.ne(1  ).to(a_  )
           UpperCamelCase__    :   Optional[Any]					=				ids_tensor([self.model_tester.batch_size]						,						self.model_tester.type_sequence_label_size  )
           UpperCamelCase__    :   Optional[Any]					=				OpenLlamaForSequenceClassification(a_  )
           model.to(a_  )
           model.eval()
           UpperCamelCase__    :   Tuple					=				model(a_						,						attention_mask=a_						,						labels=a_  )
           self.assertEqual(result.logits.shape						,						(self.model_tester.batch_size, self.model_tester.num_labels)  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__    :   Optional[int]					=				self.model_tester.prepare_config_and_inputs_for_common()
           UpperCamelCase__    :   int					=				3
           UpperCamelCase__    :   List[str]					=				"single_label_classification"
           UpperCamelCase__    :   int					=				input_dict["input_ids"]
           UpperCamelCase__    :   Tuple					=				input_ids.ne(1  ).to(a_  )
           UpperCamelCase__    :   int					=				ids_tensor([self.model_tester.batch_size]						,						self.model_tester.type_sequence_label_size  )
           UpperCamelCase__    :   List[Any]					=				OpenLlamaForSequenceClassification(a_  )
           model.to(a_  )
           model.eval()
           UpperCamelCase__    :   Union[str, Any]					=				model(a_						,						attention_mask=a_						,						labels=a_  )
           self.assertEqual(result.logits.shape						,						(self.model_tester.batch_size, self.model_tester.num_labels)  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           UpperCamelCase__    :   Union[str, Any]					=				self.model_tester.prepare_config_and_inputs_for_common()
           UpperCamelCase__    :   Any					=				3
           UpperCamelCase__    :   List[Any]					=				"multi_label_classification"
           UpperCamelCase__    :   List[str]					=				input_dict["input_ids"]
           UpperCamelCase__    :   List[Any]					=				input_ids.ne(1  ).to(a_  )
           UpperCamelCase__    :   Optional[Any]					=				ids_tensor(
               [self.model_tester.batch_size, config.num_labels]						,						self.model_tester.type_sequence_label_size  ).to(torch.float  )
           UpperCamelCase__    :   str					=				OpenLlamaForSequenceClassification(a_  )
           model.to(a_  )
           model.eval()
           UpperCamelCase__    :   Union[str, Any]					=				model(a_						,						attention_mask=a_						,						labels=a_  )
           self.assertEqual(result.logits.shape						,						(self.model_tester.batch_size, self.model_tester.num_labels)  )






     @unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''  )
     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           pass






     @parameterized.expand([('''linear''',), ('''dynamic''',)]  )
     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE  )  ->   int:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				self.model_tester.prepare_config_and_inputs_for_common()
           UpperCamelCase__    :   Dict					=				ids_tensor([1, 1_0]						,						config.vocab_size  )
           UpperCamelCase__    :   Optional[int]					=				ids_tensor([1, int(config.max_position_embeddings * 1.5  )]						,						config.vocab_size  )

           set_seed(4_2  )  # Fixed seed at init time so the two models get the same random weights
           UpperCamelCase__    :   Union[str, Any]					=				OpenLlamaModel(a_  )
           original_model.to(a_  )
           original_model.eval()
           UpperCamelCase__    :   Union[str, Any]					=				original_model(a_  ).last_hidden_state
           UpperCamelCase__    :   str					=				original_model(a_  ).last_hidden_state

           set_seed(4_2  )  # Fixed seed at init time so the two models get the same random weights
           UpperCamelCase__    :   str					=				{"type": scaling_type, "factor": 10.0}
           UpperCamelCase__    :   Optional[Any]					=				OpenLlamaModel(a_  )
           scaled_model.to(a_  )
           scaled_model.eval()
           UpperCamelCase__    :   Tuple					=				scaled_model(a_  ).last_hidden_state
           UpperCamelCase__    :   List[Any]					=				scaled_model(a_  ).last_hidden_state

           # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
           # maximum sequence length, so the outputs for the short input should match.
           if scaling_type == "dynamic":
                 self.assertTrue(torch.allclose(a_						,						a_						,						atol=1e-5  )  )
           else:
                 self.assertFalse(torch.allclose(a_						,						a_						,						atol=1e-5  )  )

           # The output should be different for long inputs
           self.assertFalse(torch.allclose(a_						,						a_						,						atol=1e-5  )  )




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCamelCase      ={"configuration_swin": ["SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP", "SwinConfig", "SwinOnnxConfig"]}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase      =[
        "SWIN_PRETRAINED_MODEL_ARCHIVE_LIST",
        "SwinForImageClassification",
        "SwinForMaskedImageModeling",
        "SwinModel",
        "SwinPreTrainedModel",
        "SwinBackbone",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase      =[
        "TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST",
        "TFSwinForImageClassification",
        "TFSwinForMaskedImageModeling",
        "TFSwinModel",
        "TFSwinPreTrainedModel",
    ]

if TYPE_CHECKING:
    from .configuration_swin import SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinConfig, SwinOnnxConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_swin import (
            SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
            SwinBackbone,
            SwinForImageClassification,
            SwinForMaskedImageModeling,
            SwinModel,
            SwinPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_swin import (
            TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFSwinForImageClassification,
            TFSwinForMaskedImageModeling,
            TFSwinModel,
            TFSwinPreTrainedModel,
        )

else:
    import sys

    lowerCamelCase      =_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


