


import tempfile
import unittest

from transformers import TaConfig, is_torch_available
from transformers.testing_utils import (
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import AutoTokenizer, UMTaForConditionalGeneration, UMTaForQuestionAnswering, UMTaModel







class     _lowercase				:


       """simple docstring"""




       def __init__(     self							:    Union[str, Any]      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    Union[str, Any]=99      ,   __lowerCamelCase							:    Dict=13      ,   __lowerCamelCase							:    int=7      ,   __lowerCamelCase							:    Any=9      ,   __lowerCamelCase							:    Dict=True      ,   __lowerCamelCase							:    int=True      ,   __lowerCamelCase							:    Tuple=False      ,   __lowerCamelCase							:    Tuple=32      ,   __lowerCamelCase							:    Dict=5      ,   __lowerCamelCase							:    Optional[int]=4      ,   __lowerCamelCase							:    List[Any]=37      ,   __lowerCamelCase							:    Union[str, Any]=8      ,   __lowerCamelCase							:    Optional[Any]=0.1      ,   __lowerCamelCase							:    str=0.0_0_2      ,   __lowerCamelCase							:    str=1      ,   __lowerCamelCase							:    Any=0      ,   __lowerCamelCase							:    Union[str, Any]=0      ,   __lowerCamelCase							:    Dict=None      ,   __lowerCamelCase							:    Optional[Any]=None      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						parent
             lowerCamelCase__	:       Union[str, Any]					=						batch_size
             lowerCamelCase__	:       Any					=						encoder_seq_length
             lowerCamelCase__	:       str					=						decoder_seq_length
             # For common tests
             lowerCamelCase__	:       Optional[int]					=						self.decoder_seq_length
             lowerCamelCase__	:       Optional[Any]					=						is_training
             lowerCamelCase__	:       List[Any]					=						use_attention_mask
             lowerCamelCase__	:       Union[str, Any]					=						use_labels
             lowerCamelCase__	:       Any					=						vocab_size
             lowerCamelCase__	:       Optional[int]					=						hidden_size
             lowerCamelCase__	:       List[str]					=						num_hidden_layers
             lowerCamelCase__	:       Union[str, Any]					=						num_attention_heads
             lowerCamelCase__	:       Any					=						d_ff
             lowerCamelCase__	:       Any					=						relative_attention_num_buckets
             lowerCamelCase__	:       Optional[Any]					=						dropout_rate
             lowerCamelCase__	:       int					=						initializer_factor
             lowerCamelCase__	:       Optional[Any]					=						eos_token_id
             lowerCamelCase__	:       Dict					=						pad_token_id
             lowerCamelCase__	:       Optional[Any]					=						decoder_start_token_id
             lowerCamelCase__	:       Union[str, Any]					=						None
             lowerCamelCase__	:       List[str]					=						decoder_layers





       def 		lowerCAmelCase  (     self							:    Union[str, Any] ):


             '''simple docstring'''



             return TaConfig.from_pretrained("google/umt5-base" )





       def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Union[str, Any]=None      ,   __lowerCamelCase							:    Dict=None      ,   __lowerCamelCase							:    Union[str, Any]=None      ,   __lowerCamelCase							:    Optional[int]=None      ,   __lowerCamelCase							:    List[Any]=None      ,   ):


             '''simple docstring'''



             if attention_mask is None:
                   lowerCamelCase__	:       Union[str, Any]					=						input_ids.ne(config.pad_token_id )
             if decoder_attention_mask is None:
                   lowerCamelCase__	:       Any					=						decoder_input_ids.ne(config.pad_token_id )
             if head_mask is None:
                   lowerCamelCase__	:       List[Any]					=						torch.ones(config.num_hidden_layers      ,   config.num_attention_heads      ,   device=UpperCAmelCase__ )
             if decoder_head_mask is None:
                   lowerCamelCase__	:       Tuple					=						torch.ones(config.num_decoder_layers      ,   config.num_attention_heads      ,   device=UpperCAmelCase__ )
             if cross_attn_head_mask is None:
                   lowerCamelCase__	:       Union[str, Any]					=						torch.ones(
                       config.num_decoder_layers      ,   config.num_attention_heads      ,   device=UpperCAmelCase__ )
             return {
                 "input_ids": input_ids,
                 "decoder_input_ids": decoder_input_ids,
                 "attention_mask": attention_mask,
                 "decoder_attention_mask": decoder_attention_mask,
                 "head_mask": head_mask,
                 "decoder_head_mask": decoder_head_mask,
                 "cross_attn_head_mask": cross_attn_head_mask,
             }





       def 		lowerCAmelCase  (     self							:    int ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						ids_tensor([self.batch_size, self.encoder_seq_length]      ,   self.vocab_size )
             lowerCamelCase__	:       Union[str, Any]					=						ids_tensor([self.batch_size, self.decoder_seq_length]      ,   self.vocab_size )

             # we need to clamp the input ids here to avoid having pad token in between
             # this is because for NllbMoe the position_ids are prepared such that
             # all pad tokens have pos id = 2 and rest are between 2..seq_length
             # and the seq_length here is seq_length - num_pad_tokens
             # but when using past, there is no way of knowing if the past input ids had
             # pad tokens in them, which results in incorrect seq_lenth and which in turn results in
             # position_ids being off by num_pad_tokens in past input
             lowerCamelCase__	:       List[str]					=						input_ids.clamp(self.pad_token_id + 1 )
             lowerCamelCase__	:       List[str]					=						decoder_input_ids.clamp(self.pad_token_id + 1 )

             lowerCamelCase__	:       str					=						self.get_config()
             lowerCamelCase__	:       Tuple					=						config.num_attention_heads
             lowerCamelCase__	:       List[Any]					=						self.prepare_inputs_dict(UpperCAmelCase__      ,   UpperCAmelCase__      ,   UpperCAmelCase__ )
             return config, input_dict





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       List[str]					=						self.prepare_config_and_inputs()
             return config, inputs_dict





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             return TaConfig(
                 vocab_size=166      ,   d_model=self.hidden_size      ,   d_ff=self.d_ff      ,   d_kv=self.hidden_size // self.num_attention_heads      ,   num_layers=self.num_hidden_layers      ,   num_decoder_layers=self.decoder_layers      ,   num_heads=self.num_attention_heads      ,   relative_attention_num_buckets=self.relative_attention_num_buckets      ,   dropout_rate=self.dropout_rate      ,   initializer_factor=self.initializer_factor      ,   eos_token_id=self.eos_token_id      ,   bos_token_id=self.pad_token_id      ,   pad_token_id=self.pad_token_id      ,   decoder_start_token_id=self.decoder_start_token_id      ,   )





       def 		lowerCAmelCase  (     self							:    Tuple ):


             '''simple docstring'''



             return TaConfig(
                 vocab_size=self.vocab_size      ,   d_model=self.hidden_size      ,   d_ff=self.d_ff      ,   d_kv=self.hidden_size // self.num_attention_heads      ,   num_layers=self.num_hidden_layers      ,   num_decoder_layers=self.decoder_layers      ,   num_heads=self.num_attention_heads      ,   relative_attention_num_buckets=self.relative_attention_num_buckets      ,   dropout_rate=self.dropout_rate      ,   initializer_factor=self.initializer_factor      ,   eos_token_id=self.eos_token_id      ,   bos_token_id=self.pad_token_id      ,   pad_token_id=self.pad_token_id      ,   decoder_start_token_id=self.decoder_start_token_id      ,   )





       def 		lowerCAmelCase  (     self							:    int      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Tuple      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       str					=						UMTaModel(config=UpperCAmelCase__ )
             model.to(UpperCAmelCase__ )
             model.eval()
             lowerCamelCase__	:       str					=						model(
                 input_ids=UpperCAmelCase__      ,   decoder_input_ids=UpperCAmelCase__      ,   attention_mask=UpperCAmelCase__      ,   decoder_attention_mask=UpperCAmelCase__      ,   )
             lowerCamelCase__	:       int					=						model(input_ids=UpperCAmelCase__      ,   decoder_input_ids=UpperCAmelCase__ )
             lowerCamelCase__	:       int					=						result.last_hidden_state
             lowerCamelCase__	:       Dict					=						result.past_key_values
             lowerCamelCase__	:       Dict					=						result.encoder_last_hidden_state

             self.parent.assertEqual(encoder_output.size()      ,   (self.batch_size, self.encoder_seq_length, self.hidden_size) )
             self.parent.assertEqual(decoder_output.size()      ,   (self.batch_size, self.decoder_seq_length, self.hidden_size) )
             # There should be `num_layers` key value embeddings stored in decoder_past
             self.parent.assertEqual(len(UpperCAmelCase__ )      ,   config.num_layers )
             # There should be a self attn key, a self attn value, a cross attn key and a cross attn value stored in each decoder_past tuple
             self.parent.assertEqual(len(decoder_past[0] )      ,   4 )





       def 		lowerCAmelCase  (     self							:    int      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    List[str]      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       int					=						UMTaModel(config=UpperCAmelCase__ ).get_decoder().to(UpperCAmelCase__ ).eval()
             # first forward pass
             lowerCamelCase__	:       List[Any]					=						model(UpperCAmelCase__      ,   use_cache=UpperCAmelCase__ )
             lowerCamelCase__	:       List[Any]					=						model(UpperCAmelCase__ )
             lowerCamelCase__	:       Any					=						model(UpperCAmelCase__      ,   use_cache=UpperCAmelCase__ )

             self.parent.assertTrue(len(UpperCAmelCase__ ) == len(UpperCAmelCase__ ) )
             self.parent.assertTrue(len(UpperCAmelCase__ ) == len(UpperCAmelCase__ ) + 1 )

             lowerCamelCase__	:       List[str]					=						outputs.to_tuple()

             # create hypothetical next token and extent to next_input_ids
             lowerCamelCase__	:       Any					=						ids_tensor((self.batch_size, 1)      ,   config.vocab_size )

             # append to next input_ids and
             lowerCamelCase__	:       Union[str, Any]					=						torch.cat([input_ids, next_tokens]      ,   dim=-1 )

             lowerCamelCase__	:       Any					=						model(UpperCAmelCase__ )['''last_hidden_state''']
             lowerCamelCase__	:       Optional[Any]					=						model(UpperCAmelCase__      ,   past_key_values=UpperCAmelCase__ )['''last_hidden_state''']

             # select random slice
             lowerCamelCase__	:       Tuple					=						ids_tensor((1,)      ,   output_from_past.shape[-1] ).item()
             lowerCamelCase__	:       Union[str, Any]					=						output_from_no_past[:, -1, random_slice_idx].detach()
             lowerCamelCase__	:       Tuple					=						output_from_past[:, 0, random_slice_idx].detach()

             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(UpperCAmelCase__      ,   UpperCAmelCase__      ,   atol=1E-3 ) )





       def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    int      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       int					=						UMTaModel(config=UpperCAmelCase__ ).to(UpperCAmelCase__ ).half().eval()
             lowerCamelCase__	:       str					=						model(**UpperCAmelCase__ )['''last_hidden_state''']
             self.parent.assertFalse(torch.isnan(UpperCAmelCase__ ).any().item() )







@require_torch
class     _lowercase				(							lowercase__							,						lowercase__							,						lowercase__							,						unittest.TestCase):


       """simple docstring"""




       A__      		=    (
           (UMTaModel, UMTaForConditionalGeneration, UMTaForQuestionAnswering) if is_torch_available() else ()
       )
       A__      		=    (UMTaForConditionalGeneration,) if is_torch_available() else ()
       A__      		=    (
           {
               'conversational': UMTaForConditionalGeneration,
               'feature-extraction': UMTaModel,
               'summarization': UMTaForConditionalGeneration,
               'text2text-generation': UMTaForConditionalGeneration,
               'translation': UMTaForConditionalGeneration,
               'question-answering': UMTaForQuestionAnswering,
           }
           if is_torch_available()
           else {}
       )
       A__      		=    True
       A__      		=    False
       A__      		=    False
       A__      		=    True
       A__      		=    True
       # The small UMT5 model needs higher percentages for CPU/MP tests
       A__      		=    [0.8, 0.9]





       def 		lowerCAmelCase  (     self							:    Optional[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						UMTaModelTester(self )





       @unittest.skip("Test has a segmentation fault on torch 1.8.0" )
       def 		lowerCAmelCase  (     self							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						self.model_tester.prepare_config_and_inputs()
             lowerCamelCase__	:       Optional[Any]					=						UMTaModel(config_and_inputs[0] ).to(UpperCAmelCase__ )
             with tempfile.TemporaryDirectory() as tmpdirname:
                   torch.onnx.export(
                       UpperCAmelCase__      ,   (config_and_inputs[1], config_and_inputs[3], config_and_inputs[2])      ,   f"{tmpdirname}/t5_test.onnx"      ,   export_params=UpperCAmelCase__      ,   opset_version=9      ,   input_names=["input_ids", "decoder_input_ids"]      ,   )





       @unittest.skipIf(torch_device == "cpu"      ,   "Cant do half precision" )
       def 		lowerCAmelCase  (     self							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model_fpaa_forward(*UpperCAmelCase__ )





       def 		lowerCAmelCase  (     self							:    Tuple ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						['''encoder_attentions''', '''decoder_attentions''', '''cross_attentions''']
             lowerCamelCase__	:       List[Any]					=						self.model_tester.prepare_config_and_inputs()
             lowerCamelCase__	:       int					=						config_and_inputs[0]
             lowerCamelCase__	:       Union[str, Any]					=						UMTaForConditionalGeneration(UpperCAmelCase__ ).eval()
             model.to(UpperCAmelCase__ )

             lowerCamelCase__	:       str					=						{
                 '''head_mask''': torch.zeros(config.num_layers      ,   config.num_heads      ,   device=UpperCAmelCase__ ),
                 '''decoder_head_mask''': torch.zeros(config.num_decoder_layers      ,   config.num_heads      ,   device=UpperCAmelCase__ ),
                 '''cross_attn_head_mask''': torch.zeros(config.num_decoder_layers      ,   config.num_heads      ,   device=UpperCAmelCase__ ),
             }

             for attn_name, (name, mask) in zip(UpperCAmelCase__      ,   head_masking.items() ):
                   lowerCamelCase__	:       int					=						{name: mask}
                   # Explicitly pass decoder_head_mask as it is required from T5 model when head_mask specified
                   if name == "head_mask":
                         lowerCamelCase__	:       List[str]					=						torch.ones(
                             config.num_decoder_layers      ,   config.num_heads      ,   device=UpperCAmelCase__ )

                   lowerCamelCase__	:       Union[str, Any]					=						model.generate(
                       config_and_inputs[1]["input_ids"]      ,   num_beams=1      ,   max_length=3      ,   output_attentions=UpperCAmelCase__      ,   return_dict_in_generate=UpperCAmelCase__      ,   **UpperCAmelCase__      ,   )
                   # We check the state of decoder_attentions and cross_attentions just from the last step
                   lowerCamelCase__	:       List[str]					=						out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1]
                   self.assertEqual(sum([w.sum().item() for w in attn_weights] )      ,   0.0 )





       @unittest.skip("Does not work on the tiny model as we keep hitting edge cases." )
       def 		lowerCAmelCase  (     self							:    Any ):


             '''simple docstring'''



             pass







@require_torch
@require_sentencepiece
@require_tokenizers
class     _lowercase				(							unittest.TestCase):


       """simple docstring"""




       @slow
       @unittest.skip(
           "Unless we stop stripping left and right by default for all special tokens, the expected ids obtained here will not match the original ones. Wait for https://github.com/huggingface/transformers/pull/23909 to be merged" )
       def 		lowerCAmelCase  (     self							:    int ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[Any]					=						UMTaForConditionalGeneration.from_pretrained("google/umt5-small"      ,   return_dict=UpperCAmelCase__ ).to(UpperCAmelCase__ )
             lowerCamelCase__	:       int					=						AutoTokenizer.from_pretrained("google/umt5-small"      ,   use_fast=UpperCAmelCase__      ,   legacy=UpperCAmelCase__ )
             lowerCamelCase__	:       List[str]					=						[
                 '''Bonjour monsieur <extra_id_0> bien <extra_id_1>.''',
                 '''No se como puedo <extra_id_0>.''',
                 '''This is the reason why we <extra_id_0> them.''',
                 '''The <extra_id_0> walks in <extra_id_1>, seats''',
                 '''A <extra_id_0> walks into a bar and orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.''',
             ]
             lowerCamelCase__	:       Dict					=						tokenizer(UpperCAmelCase__      ,   return_tensors="pt"      ,   padding=UpperCAmelCase__ ).input_ids
             # fmt: off
             lowerCamelCase__	:       Optional[Any]					=						torch.tensor(
                 [
                     [ 38530, 210703, 256299, 1410, 256298, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
                     [   826, 321, 671, 25922, 256299, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
                     [  1460, 339, 312, 19014, 10620, 758, 256299, 2355,274, 1, 0, 0, 0, 0, 0, 0,0, 0],
                     [   517, 256299, 14869, 281, 301, 256298, 275, 119983,1, 0, 0, 0, 0, 0, 0, 0,0, 0],
                     [   320, 256299, 14869, 281, 2234, 289, 2275, 333,61391, 289, 256298, 543, 256297, 168714, 329, 256296,274, 1],
                 ] )
             # fmt: on
             torch.testing.assert_allclose(UpperCAmelCase__      ,   UpperCAmelCase__ )

             lowerCamelCase__	:       List[Any]					=						model.generate(input_ids.to(UpperCAmelCase__ ) )
             lowerCamelCase__	:       int					=						[
                 '''<pad><extra_id_0> et<extra_id_1> [eod] <extra_id_2><extra_id_55>.. [eod] 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 <extra_id_56>ajšietosto<extra_id_56>lleux<extra_id_19><extra_id_6>ajšie</s>''',
                 '''<pad><extra_id_0>.<extra_id_1>.,<0x0A>...spech <0x0A><extra_id_20> <extra_id_21></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
                 '''<pad><extra_id_0> are not going to be a part of the world. We are not going to be a part of<extra_id_1> and<extra_id_2><0x0A><extra_id_48>.<extra_id_48></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
                 '''<pad><extra_id_0> door<extra_id_1>, the door<extra_id_2> 피해[/</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
                 '''<pad><extra_id_0>nyone who<extra_id_1> drink<extra_id_2> a<extra_id_3> alcohol<extra_id_4> A<extra_id_5> A. This<extra_id_6> I<extra_id_7><extra_id_52><extra_id_53></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
             ]
             lowerCamelCase__	:       Tuple					=						tokenizer.batch_decode(UpperCAmelCase__ )
             self.assertEqual(UpperCAmelCase__      ,   UpperCAmelCase__ )






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
        OpenAIGPTConfig,
        OpenAIGPTDoubleHeadsModel,
        OpenAIGPTForSequenceClassification,
        OpenAIGPTLMHeadModel,
        OpenAIGPTModel,
    )







class     _lowercase				:


       """simple docstring"""




       def __init__(     self							:    Dict      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Optional[int]=13      ,   __lowerCamelCase							:    List[str]=7      ,   __lowerCamelCase							:    Tuple=True      ,   __lowerCamelCase							:    Optional[int]=True      ,   __lowerCamelCase							:    List[str]=True      ,   __lowerCamelCase							:    Union[str, Any]=99      ,   __lowerCamelCase							:    List[Any]=32      ,   __lowerCamelCase							:    List[Any]=5      ,   __lowerCamelCase							:    Optional[Any]=4      ,   __lowerCamelCase							:    Optional[int]=37      ,   __lowerCamelCase							:    List[str]="gelu"      ,   __lowerCamelCase							:    List[str]=0.1      ,   __lowerCamelCase							:    int=0.1      ,   __lowerCamelCase							:    List[str]=512      ,   __lowerCamelCase							:    Optional[Any]=16      ,   __lowerCamelCase							:    Optional[Any]=2      ,   __lowerCamelCase							:    str=0.0_2      ,   __lowerCamelCase							:    List[str]=3      ,   __lowerCamelCase							:    Tuple=4      ,   __lowerCamelCase							:    Optional[int]=None      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						parent
             lowerCamelCase__	:       int					=						batch_size
             lowerCamelCase__	:       List[Any]					=						seq_length
             lowerCamelCase__	:       Union[str, Any]					=						is_training
             lowerCamelCase__	:       Any					=						use_token_type_ids
             lowerCamelCase__	:       Union[str, Any]					=						use_labels
             lowerCamelCase__	:       List[str]					=						vocab_size
             lowerCamelCase__	:       Union[str, Any]					=						hidden_size
             lowerCamelCase__	:       List[Any]					=						num_hidden_layers
             lowerCamelCase__	:       Optional[Any]					=						num_attention_heads
             lowerCamelCase__	:       Any					=						intermediate_size
             lowerCamelCase__	:       str					=						hidden_act
             lowerCamelCase__	:       str					=						hidden_dropout_prob
             lowerCamelCase__	:       Any					=						attention_probs_dropout_prob
             lowerCamelCase__	:       List[str]					=						max_position_embeddings
             lowerCamelCase__	:       Optional[int]					=						type_vocab_size
             lowerCamelCase__	:       List[Any]					=						type_sequence_label_size
             lowerCamelCase__	:       List[str]					=						initializer_range
             lowerCamelCase__	:       List[str]					=						num_labels
             lowerCamelCase__	:       List[Any]					=						num_choices
             lowerCamelCase__	:       Optional[Any]					=						scope
             lowerCamelCase__	:       List[Any]					=						self.vocab_size - 1





       def 		lowerCAmelCase  (     self							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.vocab_size )

             lowerCamelCase__	:       Optional[Any]					=						None
             if self.use_token_type_ids:
                   lowerCamelCase__	:       Optional[Any]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.type_vocab_size )

             lowerCamelCase__	:       Any					=						None
             lowerCamelCase__	:       str					=						None
             lowerCamelCase__	:       str					=						None
             if self.use_labels:
                   lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size]      ,   self.type_sequence_label_size )
                   lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.num_labels )
                   lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size]      ,   self.num_choices )

             lowerCamelCase__	:       Union[str, Any]					=						OpenAIGPTConfig(
                 vocab_size=self.vocab_size      ,   n_embd=self.hidden_size      ,   n_layer=self.num_hidden_layers      ,   n_head=self.num_attention_heads      ,   n_positions=self.max_position_embeddings      ,   pad_token_id=self.pad_token_id      ,   )

             lowerCamelCase__	:       Optional[int]					=						ids_tensor([self.num_hidden_layers, self.num_attention_heads]      ,   2 )

             return (
                 config,
                 input_ids,
                 head_mask,
                 token_type_ids,
                 sequence_labels,
                 token_labels,
                 choice_labels,
             )





       def 		lowerCAmelCase  (     self							:    str      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    int      ,   *__lowerCamelCase							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						OpenAIGPTModel(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       Tuple					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   head_mask=__lowerCamelCase )
             lowerCamelCase__	:       str					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase )
             lowerCamelCase__	:       Optional[int]					=						model(__lowerCamelCase )

             self.parent.assertEqual(result.last_hidden_state.shape      ,   (self.batch_size, self.seq_length, self.hidden_size) )





       def 		lowerCAmelCase  (     self							:    str      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    Any      ,   *__lowerCamelCase							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						OpenAIGPTLMHeadModel(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       List[str]					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.loss.shape      ,   () )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size) )





       def 		lowerCAmelCase  (     self							:    Dict      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Optional[int]      ,   *__lowerCamelCase							:    Tuple ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						OpenAIGPTDoubleHeadsModel(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       Optional[Any]					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.loss.shape      ,   () )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size) )





       def 		lowerCAmelCase  (     self							:    Tuple      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    List[Any]      ,   *__lowerCamelCase							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Dict					=						self.num_labels
             lowerCamelCase__	:       Tuple					=						OpenAIGPTForSequenceClassification(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size]      ,   self.type_sequence_label_size )
             lowerCamelCase__	:       List[str]					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_labels) )





       def 		lowerCAmelCase  (     self							:    List[str] ):


             '''simple docstring'''



             lowerCamelCase__	:       str					=						self.prepare_config_and_inputs()
             (
                 (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     
             )	:       Any					=						config_and_inputs
             lowerCamelCase__	:       Union[str, Any]					=						{
                 "input_ids": input_ids,
                 "token_type_ids": token_type_ids,
                 "head_mask": head_mask,
             }

             return config, inputs_dict







@require_torch
class     _lowercase				(							lowercase__							,						lowercase__							,						lowercase__							,						unittest.TestCase):


       """simple docstring"""




       A__      		=    (
           (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
           if is_torch_available()
           else ()
       )
       A__      		=    (
           (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
       )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
       A__      		=    (
           {
               "feature-extraction": OpenAIGPTModel,
               "text-classification": OpenAIGPTForSequenceClassification,
               "text-generation": OpenAIGPTLMHeadModel,
               "zero-shot": OpenAIGPTForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )





       def 		lowerCAmelCase  (     self							:    List[str]      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Union[str, Any] ):


             '''simple docstring'''



             if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                   # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                   # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
                   # tiny config could not be created.
                   return True

             return False





       def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    Tuple=False ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						super()._prepare_for_class(__lowerCamelCase      ,   __lowerCamelCase      ,   return_labels=__lowerCamelCase )

             if return_labels:
                   if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                         lowerCamelCase__	:       Optional[Any]					=						torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)      ,   dtype=torch.long      ,   device=__lowerCamelCase      ,   )
                         lowerCamelCase__	:       Tuple					=						inputs_dict["labels"]
                         lowerCamelCase__	:       Any					=						inputs_dict["labels"]
                         lowerCamelCase__	:       Any					=						torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.num_choices)      ,   dtype=torch.long      ,   device=__lowerCamelCase      ,   )
                         lowerCamelCase__	:       Union[str, Any]					=						torch.zeros(
                             self.model_tester.batch_size      ,   dtype=torch.long      ,   device=__lowerCamelCase )
             return inputs_dict





       def 		lowerCAmelCase  (     self							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						OpenAIGPTModelTester(self )
             lowerCamelCase__	:       Union[str, Any]					=						ConfigTester(self      ,   config_class=__lowerCamelCase      ,   n_embd=37 )





       def 		lowerCAmelCase  (     self							:    int ):


             '''simple docstring'''



             self.config_tester.run_common_tests()





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_openai_gpt_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    str ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_lm_head_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_double_lm_head_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__lowerCamelCase )





       @slow
       def 		lowerCAmelCase  (     self							:    List[str] ):


             '''simple docstring'''



             for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   lowerCamelCase__	:       Any					=						OpenAIGPTModel.from_pretrained(__lowerCamelCase )
                   self.assertIsNotNone(__lowerCamelCase )







@require_torch
class     _lowercase				(							unittest.TestCase):


       """simple docstring"""




       @slow
       def 		lowerCAmelCase  (     self							:    Any ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						OpenAIGPTLMHeadModel.from_pretrained("openai-gpt" )
             model.to(__lowerCamelCase )
             lowerCamelCase__	:       int					=						torch.tensor([[481, 4735, 544]]      ,   dtype=torch.long      ,   device=__lowerCamelCase )  # the president is
             lowerCamelCase__	:       Union[str, Any]					=						[
                 481,
                 4735,
                 544,
                 246,
                 963,
                 870,
                 762,
                 239,
                 244,
                 40477,
                 244,
                 249,
                 719,
                 881,
                 487,
                 544,
                 240,
                 244,
                 603,
                 481,
             ]  # the president is a very good man. " \n " i\'m sure he is, " said the

             lowerCamelCase__	:       int					=						model.generate(__lowerCamelCase      ,   do_sample=__lowerCamelCase )
             self.assertListEqual(output_ids[0].tolist()      ,   __lowerCamelCase )


