
'''simple docstring'''






import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    GPTNeoXForCausalLM,
					    GPTNeoXForQuestionAnswering,
					    GPTNeoXForSequenceClassification,
					    GPTNeoXForTokenClassification,
					    GPTNeoXModel,
					)




class     __snake_case		:
					def __init__(     self,    A,    A=13,    A=7,    A=True,    A=True,    A=True,    A=True,    A=99,    A=64,    A=5,    A=4,    A=37,    A="gelu",    A=0.1,    A=0.1,    A=512,    A=16,    A=2,    A=0.02,    A=3,    A=4,    A=None,    ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= parent
									lowerCamelCase					:			Dict									= batch_size
									lowerCamelCase					:			List[Any]									= seq_length
									lowerCamelCase					:			Tuple									= is_training
									lowerCamelCase					:			List[str]									= use_input_mask
									lowerCamelCase					:			Tuple									= use_token_type_ids
									lowerCamelCase					:			List[Any]									= use_labels
									lowerCamelCase					:			Optional[int]									= vocab_size
									lowerCamelCase					:			Optional[int]									= hidden_size
									lowerCamelCase					:			Dict									= num_hidden_layers
									lowerCamelCase					:			Tuple									= num_attention_heads
									lowerCamelCase					:			Optional[int]									= intermediate_size
									lowerCamelCase					:			Optional[int]									= hidden_act
									lowerCamelCase					:			Any									= hidden_dropout_prob
									lowerCamelCase					:			Optional[int]									= attention_probs_dropout_prob
									lowerCamelCase					:			Any									= max_position_embeddings
									lowerCamelCase					:			str									= type_vocab_size
									lowerCamelCase					:			List[Any]									= type_sequence_label_size
									lowerCamelCase					:			Optional[Any]									= initializer_range
									lowerCamelCase					:			Tuple									= num_labels
									lowerCamelCase					:			Dict									= num_choices
									lowerCamelCase					:			Optional[Any]									= scope
									lowerCamelCase					:			Tuple									= vocab_size - 1
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			List[Any]									= ids_tensor([self.batch_size, self.seq_length],    self.vocab_size      )

									lowerCamelCase					:			Tuple									= None
									if self.use_input_mask:
													lowerCamelCase					:			Dict									= random_attention_mask([self.batch_size, self.seq_length]      )

									lowerCamelCase					:			int									= None
									if self.use_labels:
													lowerCamelCase					:			Tuple									= ids_tensor([self.batch_size, self.seq_length],    self.num_labels      )

									lowerCamelCase					:			Any									= self.get_config()

									return config, input_ids, input_mask, token_labels
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									return GPTNeoXConfig(
									    vocab_size=self.vocab_size,    hidden_size=self.hidden_size,    num_hidden_layers=self.num_hidden_layers,    num_attention_heads=self.num_attention_heads,    intermediate_size=self.intermediate_size,    hidden_act=self.hidden_act,    hidden_dropout_prob=self.hidden_dropout_prob,    attention_probs_dropout_prob=self.attention_probs_dropout_prob,    max_position_embeddings=self.max_position_embeddings,    type_vocab_size=self.type_vocab_size,    is_decoder=A,    initializer_range=self.initializer_range,    pad_token_id=self.pad_token_id,    )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Dict									= self.prepare_config_and_inputs()

									lowerCamelCase					:			List[Any]									= True

									return config, input_ids, input_mask, token_labels
					def     UpperCAmelCase_							(     self,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			Dict									= GPTNeoXModel(config=A      )
									model.to(A      )
									model.eval()
									lowerCamelCase					:			str									= model(A,    attention_mask=A      )
									lowerCamelCase					:			Any									= model(A      )
									self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.hidden_size)      )
					def     UpperCAmelCase_							(     self,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			List[Any]									= True
									lowerCamelCase					:			Optional[int]									= GPTNeoXModel(A      )
									model.to(A      )
									model.eval()
									lowerCamelCase					:			List[Any]									= model(A,    attention_mask=A      )
									self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.hidden_size)      )
					def     UpperCAmelCase_							(     self,    A,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= GPTNeoXForCausalLM(config=A      )
									model.to(A      )
									model.eval()
									lowerCamelCase					:			Union[str, Any]									= model(A,    attention_mask=A,    labels=A      )
									self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.vocab_size)      )
					def     UpperCAmelCase_							(     self,    A,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[int]									= self.num_labels
									lowerCamelCase					:			List[str]									= GPTNeoXForQuestionAnswering(A      )
									model.to(A      )
									model.eval()
									lowerCamelCase					:			Optional[Any]									= model(A,    attention_mask=A      )
									self.parent.assertEqual(result.start_logits.shape,    (self.batch_size, self.seq_length)      )
									self.parent.assertEqual(result.end_logits.shape,    (self.batch_size, self.seq_length)      )
					def     UpperCAmelCase_							(     self,    A,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[int]									= self.num_labels
									lowerCamelCase					:			Union[str, Any]									= GPTNeoXForSequenceClassification(A      )
									model.to(A      )
									model.eval()
									lowerCamelCase					:			List[Any]									= ids_tensor([self.batch_size],    self.type_sequence_label_size      )
									lowerCamelCase					:			Any									= model(A,    attention_mask=A,    labels=A      )
									self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.num_labels)      )
					def     UpperCAmelCase_							(     self,    A,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			str									= self.num_labels
									lowerCamelCase					:			Union[str, Any]									= GPTNeoXForTokenClassification(A      )
									model.to(A      )
									model.eval()
									lowerCamelCase					:			Dict									= model(A,    attention_mask=A,    labels=A      )
									self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.num_labels)      )
					def     UpperCAmelCase_							(     self,    A,    A,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			str									= True
									lowerCamelCase					:			str									= GPTNeoXForCausalLM(config=A      )
									model.to(A      )
									model.eval()

									# first forward pass
									lowerCamelCase					:			int									= model(A,    attention_mask=A,    use_cache=A      )
									lowerCamelCase					:			List[Any]									= outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									lowerCamelCase					:			List[str]									= ids_tensor((self.batch_size, 3),    config.vocab_size      )
									lowerCamelCase					:			Optional[int]									= ids_tensor((self.batch_size, 3),    vocab_size=2      )

									# append to next input_ids and
									lowerCamelCase					:			Tuple									= torch.cat([input_ids, next_tokens],    dim=-1      )
									lowerCamelCase					:			List[str]									= torch.cat([input_mask, next_mask],    dim=-1      )

									lowerCamelCase					:			str									= model(A,    attention_mask=A,    output_hidden_states=A      )
									lowerCamelCase					:			List[str]									= output_from_no_past['hidden_states'][0]
									lowerCamelCase					:			Tuple									= model(
									    A,    attention_mask=A,    past_key_values=A,    output_hidden_states=A,    )['hidden_states'][0]

									# select random slice
									lowerCamelCase					:			List[str]									= ids_tensor((1,),    output_from_past.shape[-1]      ).item()
									lowerCamelCase					:			List[str]									= output_from_no_past[:, -3:, random_slice_idx].detach()
									lowerCamelCase					:			Any									= output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(A,    A,    atol=1e-3      )      )




					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			List[str]									= self.prepare_config_and_inputs()
									lowerCamelCase					:			Any									= config_and_inputs
									lowerCamelCase					:			Union[str, Any]									= {'input_ids': input_ids, 'attention_mask': input_mask}
									return config, inputs_dict





@require_torch
class     __snake_case		(   a__    ,   a__    ,   a__    ,   unittest.TestCase):
					_lowerCAmelCase			     =	(
					    (
					        GPTNeoXModel,
					        GPTNeoXForCausalLM,
					        GPTNeoXForQuestionAnswering,
					        GPTNeoXForSequenceClassification,
					        GPTNeoXForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					_lowerCAmelCase			     =	(GPTNeoXForCausalLM,) if is_torch_available() else ()
					_lowerCAmelCase			     =	(
					    {
					        '''feature-extraction''': GPTNeoXModel,
					        '''question-answering''': GPTNeoXForQuestionAnswering,
					        '''text-classification''': GPTNeoXForSequenceClassification,
					        '''text-generation''': GPTNeoXForCausalLM,
					        '''token-classification''': GPTNeoXForTokenClassification,
					        '''zero-shot''': GPTNeoXForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					_lowerCAmelCase			     =	False
					_lowerCAmelCase			     =	False
					_lowerCAmelCase			     =	False
					_lowerCAmelCase			     =	False
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= GPTNeoXModelTester(self      )
									lowerCamelCase					:			Dict									= ConfigTester(self,    config_class=A,    hidden_size=64,    num_attention_heads=8      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									self.config_tester.run_common_tests()
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[Any]									= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(A,    A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Dict									= self.model_tester.prepare_config_and_inputs_for_decoder()
									self.model_tester.create_and_check_model_as_decoder(A,    A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			List[Any]									= self.model_tester.prepare_config_and_inputs_for_decoder()

									lowerCamelCase					:			Union[str, Any]									= None

									self.model_tester.create_and_check_model_as_decoder(A,    A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_decoder_model_past_large_inputs(A,    A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[int]									= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_causal_lm(*A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			List[Any]									= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[int]									= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Tuple									= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*A      )
					@unittest.skip(reason='Feed forward chunking is not implemented'      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									pass




					@parameterized.expand([('linear',), ('dynamic',)]      )
					def     UpperCAmelCase_							(     self,    A      ):

									"""simple docstring"""


									lowerCamelCase					:			Dict									= self.model_tester.prepare_config_and_inputs_for_common()
									lowerCamelCase					:			Dict									= ids_tensor([1, 10],    config.vocab_size      )
									lowerCamelCase					:			Union[str, Any]									= ids_tensor([1, int(config.max_position_embeddings * 1.5      )],    config.vocab_size      )

									set_seed(42      )  # Fixed seed at init time so the two models get the same random weights
									lowerCamelCase					:			Dict									= GPTNeoXModel(A      )
									original_model.to(A      )
									original_model.eval()
									lowerCamelCase					:			Dict									= original_model(A      ).last_hidden_state
									lowerCamelCase					:			Dict									= original_model(A      ).last_hidden_state

									set_seed(42      )  # Fixed seed at init time so the two models get the same random weights
									lowerCamelCase					:			Dict									= {'type': scaling_type, 'factor': 10.0}
									lowerCamelCase					:			Tuple									= GPTNeoXModel(A      )
									scaled_model.to(A      )
									scaled_model.eval()
									lowerCamelCase					:			Optional[int]									= scaled_model(A      ).last_hidden_state
									lowerCamelCase					:			List[Any]									= scaled_model(A      ).last_hidden_state

									# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
									# maximum sequence length, so the outputs for the short input should match.
									if scaling_type == "dynamic":
													self.assertTrue(torch.allclose(A,    A,    atol=1e-5      )      )
									else:
													self.assertFalse(torch.allclose(A,    A,    atol=1e-5      )      )

									# The output should be different for long inputs
									self.assertFalse(torch.allclose(A,    A,    atol=1e-5      )      )





@require_torch
class     __snake_case		(   unittest.TestCase):
					@slow
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Tuple									= AutoTokenizer.from_pretrained('EleutherAI/pythia-410m-deduped'      )
									for checkpointing in [True, False]:
													lowerCamelCase					:			Any									= GPTNeoXForCausalLM.from_pretrained('EleutherAI/pythia-410m-deduped'      )

													if checkpointing:
																	model.gradient_checkpointing_enable()
													else:
																	model.gradient_checkpointing_disable()
													model.to(A      )

													lowerCamelCase					:			str									= tokenizer('My favorite food is',    return_tensors='pt'      ).to(A      )
													# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
													# See: https://github.com/huggingface/transformers/pull/24193
													lowerCamelCase					:			Optional[Any]									= 'My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'

													lowerCamelCase					:			str									= model.generate(**A,    do_sample=A,    max_new_tokens=20      )
													lowerCamelCase					:			List[Any]									= tokenizer.batch_decode(A      )[0]

													self.assertEqual(A,    A      )





'''simple docstring'''






import numpy as np





def    UpperCAmelCase							(  UpperCAmelCase__							:							np.array):
				return 1 / (1 + np.exp(-vector))


if __name__ == "__main__":
					import doctest

					doctest.testmod()
