



import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				lowercase__   						=						MODEL_FOR_MASKED_LM_MAPPING
				lowercase__   						=						TF_MODEL_FOR_MASKED_LM_MAPPING

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									super().tearDown()
									# clean-up as much as possible GPU memory occupied by PyTorch
									gc.collect()
									if is_torch_available():
														import torch

														torch.cuda.empty_cache()

				@require_tf
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline(task="fill-mask"     , model="sshleifer/tiny-distilroberta-base"     , top_k=2     , framework="tf"    )
									UpperCamelCase_		  =   unmasker("My name is <mask>"    )
									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        {"sequence": "My name is grouped", "score": 2.1e-05, "token": 3_8015, "token_str": " grouped"},
									        {"sequence": "My name is accuser", "score": 2.1e-05, "token": 2_5506, "token_str": " accuser"},
									    ]     , )

									UpperCamelCase_		  =   unmasker("The largest city in France is <mask>"    )
									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        {
									            "sequence": "The largest city in France is grouped",
									            "score": 2.1e-05,
									            "token": 3_8015,
									            "token_str": " grouped",
									        },
									        {
									            "sequence": "The largest city in France is accuser",
									            "score": 2.1e-05,
									            "token": 2_5506,
									            "token_str": " accuser",
									        },
									    ]     , )

									UpperCamelCase_		  =   unmasker("My name is <mask>"     , targets=[" Patrick", " Clara", " Teven"]     , top_k=3    )
									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        {"sequence": "My name is Clara", "score": 2e-05, "token": 1_3606, "token_str": " Clara"},
									        {"sequence": "My name is Patrick", "score": 2e-05, "token": 3499, "token_str": " Patrick"},
									        {"sequence": "My name is Te", "score": 1.9e-05, "token": 2941, "token_str": " Te"},
									    ]     , )

				@require_torch
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline(task="fill-mask"     , model="sshleifer/tiny-distilroberta-base"     , top_k=2     , framework="pt"    )

									UpperCamelCase_		  =   unmasker("My name is <mask>"    )
									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        {"sequence": "My name is Maul", "score": 2.2e-05, "token": 3_5676, "token_str": " Maul"},
									        {"sequence": "My name isELS", "score": 2.2e-05, "token": 1_6416, "token_str": "ELS"},
									    ]     , )

									UpperCamelCase_		  =   unmasker("The largest city in France is <mask>"    )
									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        {
									            "sequence": "The largest city in France is Maul",
									            "score": 2.2e-05,
									            "token": 3_5676,
									            "token_str": " Maul",
									        },
									        {"sequence": "The largest city in France isELS", "score": 2.2e-05, "token": 1_6416, "token_str": "ELS"},
									    ]     , )

									UpperCamelCase_		  =   unmasker("My name is <mask>"     , targets=[" Patrick", " Clara", " Teven"]     , top_k=3    )
									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        {"sequence": "My name is Patrick", "score": 2.1e-05, "token": 3499, "token_str": " Patrick"},
									        {"sequence": "My name is Te", "score": 2e-05, "token": 2941, "token_str": " Te"},
									        {"sequence": "My name is Clara", "score": 2e-05, "token": 1_3606, "token_str": " Clara"},
									    ]     , )

									UpperCamelCase_		  =   unmasker("My name is <mask> <mask>"     , top_k=2    )

									self.assertEqual(
									    nested_simplify(snake_case__     , decimals=6    )     , [
									        [
									            {
									                "score": 2.2e-05,
									                "token": 3_5676,
									                "token_str": " Maul",
									                "sequence": "<s>My name is Maul<mask></s>",
									            },
									            {"score": 2.2e-05, "token": 1_6416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
									        ],
									        [
									            {
									                "score": 2.2e-05,
									                "token": 3_5676,
									                "token_str": " Maul",
									                "sequence": "<s>My name is<mask> Maul</s>",
									            },
									            {"score": 2.2e-05, "token": 1_6416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
									        ],
									    ]     , )

				@require_torch_gpu
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline("fill-mask"     , model="hf-internal-testing/tiny-random-distilbert"     , device=0     , framework="pt"    )

									# convert model to fp16
									pipe.model.half()

									UpperCamelCase_		  =   pipe("Paris is the [MASK] of France."    )
									# We actually don't care about the result, we just want to make sure
									# it works, meaning the float16 tensor got casted back to float32
									# for postprocessing.
									self.assertIsInstance(snake_case__     , snake_case__    )

				@slow
				@require_torch
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline(task="fill-mask"     , model="distilroberta-base"     , top_k=2     , framework="pt"    )
									self.run_large_test(snake_case__    )

				@slow
				@require_tf
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline(task="fill-mask"     , model="distilroberta-base"     , top_k=2     , framework="tf"    )
									self.run_large_test(snake_case__    )

				def       _lowerCamelCase					( self     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   unmasker("My name is <mask>"    )
									self.assertEqual(
									    nested_simplify(snake_case__    )     , [
									        {"sequence": "My name is John", "score": 0.008, "token": 610, "token_str": " John"},
									        {"sequence": "My name is Chris", "score": 0.007, "token": 1573, "token_str": " Chris"},
									    ]     , )
									UpperCamelCase_		  =   unmasker("The largest city in France is <mask>"    )
									self.assertEqual(
									    nested_simplify(snake_case__    )     , [
									        {
									            "sequence": "The largest city in France is Paris",
									            "score": 0.251,
									            "token": 2201,
									            "token_str": " Paris",
									        },
									        {
									            "sequence": "The largest city in France is Lyon",
									            "score": 0.214,
									            "token": 1_2790,
									            "token_str": " Lyon",
									        },
									    ]     , )

									UpperCamelCase_		  =   unmasker("My name is <mask>"     , targets=[" Patrick", " Clara", " Teven"]     , top_k=3    )
									self.assertEqual(
									    nested_simplify(snake_case__    )     , [
									        {"sequence": "My name is Patrick", "score": 0.005, "token": 3499, "token_str": " Patrick"},
									        {"sequence": "My name is Clara", "score": 0.000, "token": 1_3606, "token_str": " Clara"},
									        {"sequence": "My name is Te", "score": 0.000, "token": 2941, "token_str": " Te"},
									    ]     , )

				@require_torch
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline(task="fill-mask"     , model="sshleifer/tiny-distilroberta-base"     , framework="pt"    )
									UpperCamelCase_		  =   None
									UpperCamelCase_		  =   None
									self.run_pipeline_test(snake_case__     , []    )

				@require_tf
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   pipeline(task="fill-mask"     , model="sshleifer/tiny-distilroberta-base"     , framework="tf"    )
									UpperCamelCase_		  =   None
									UpperCamelCase_		  =   None
									self.run_pipeline_test(snake_case__     , []    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									if tokenizer is None or tokenizer.mask_token_id is None:
														self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)"    )

									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__    )
									UpperCamelCase_		  =   [
									    F"""This is another {tokenizer.mask_token} test""",
									]
									return fill_masker, examples

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   fill_masker.tokenizer
									UpperCamelCase_		  =   fill_masker.model

									UpperCamelCase_		  =   fill_masker(
									    F"""This is a {tokenizer.mask_token}"""     , )
									self.assertEqual(
									    snake_case__     , [
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									    ]     , )

									UpperCamelCase_		  =   fill_masker([F"""This is a {tokenizer.mask_token}"""]    )
									self.assertEqual(
									    snake_case__     , [
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									    ]     , )

									UpperCamelCase_		  =   fill_masker([F"""This is a {tokenizer.mask_token}""", F"""Another {tokenizer.mask_token} great test."""]    )
									self.assertEqual(
									    snake_case__     , [
									        [
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        ],
									        [
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        ],
									    ]     , )

									with self.assertRaises(snake_case__    ):
														fill_masker([None]    )
									# No mask_token is not supported
									with self.assertRaises(snake_case__    ):
														fill_masker("This is"    )

									self.run_test_top_k(snake_case__     , snake_case__    )
									self.run_test_targets(snake_case__     , snake_case__    )
									self.run_test_top_k_targets(snake_case__     , snake_case__    )
									self.fill_mask_with_duplicate_targets_and_top_k(snake_case__     , snake_case__    )
									self.fill_mask_with_multiple_masks(snake_case__     , snake_case__    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   tokenizer.get_vocab()
									UpperCamelCase_		  =   sorted(vocab.keys()    )[:2]
									# Pipeline argument
									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__     , targets=snake_case__    )
									UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""    )
									self.assertEqual(
									    snake_case__     , [
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									    ]     , )
									UpperCamelCase_		  =   {vocab[el] for el in targets}
									self.assertEqual({el["token"] for el in outputs}     , snake_case__    )
									UpperCamelCase_		  =   [tokenizer.decode([x]    ) for x in target_ids]
									self.assertEqual({el["token_str"] for el in outputs}     , set(snake_case__    )    )

									# Call argument
									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__    )
									UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , targets=snake_case__    )
									self.assertEqual(
									    snake_case__     , [
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									    ]     , )
									UpperCamelCase_		  =   {vocab[el] for el in targets}
									self.assertEqual({el["token"] for el in outputs}     , snake_case__    )
									UpperCamelCase_		  =   [tokenizer.decode([x]    ) for x in target_ids]
									self.assertEqual({el["token_str"] for el in outputs}     , set(snake_case__    )    )

									# Score equivalence
									UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , targets=snake_case__    )
									UpperCamelCase_		  =   [top_mask["token_str"] for top_mask in outputs]
									UpperCamelCase_		  =   [top_mask["score"] for top_mask in outputs]

									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(snake_case__    ) == set(snake_case__    ):
														UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , targets=snake_case__    )
														UpperCamelCase_		  =   [top_mask["score"] for top_mask in unmasked_targets]
														self.assertEqual(nested_simplify(snake_case__    )     , nested_simplify(snake_case__    )    )

									# Raises with invalid
									with self.assertRaises(snake_case__    ):
														UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , targets=[]    )
									# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
									if "" not in tokenizer.get_vocab():
														with self.assertRaises(snake_case__    ):
																			UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , targets=[""]    )
														with self.assertRaises(snake_case__    ):
																			UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , targets=""    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__     , top_k=2    )
									UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""    )
									self.assertEqual(
									    snake_case__     , [
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									    ]     , )

									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__    )
									UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , top_k=2    )
									self.assertEqual(
									    snake_case__     , [
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									    ]     , )
									self.assertEqual(nested_simplify(snake_case__    )     , nested_simplify(snake_case__    )    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   tokenizer.get_vocab()
									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__    )

									# top_k=2, ntargets=3
									UpperCamelCase_		  =   sorted(vocab.keys()    )[:3]
									UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , top_k=2     , targets=snake_case__    )

									# If we use the most probably targets, and filter differently, we should still
									# have the same results
									UpperCamelCase_		  =   [el["token_str"] for el in sorted(snake_case__     , key=lambda snake_case__    : x["score"]     , reverse=snake_case__    )]
									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(snake_case__    ).issubset(snake_case__    ):
														UpperCamelCase_		  =   fill_masker(F"""This is a {tokenizer.mask_token}"""     , top_k=3     , targets=snake_case__    )
														# They should yield exactly the same result
														self.assertEqual(nested_simplify(snake_case__    )     , nested_simplify(snake_case__    )    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__    )
									UpperCamelCase_		  =   tokenizer.get_vocab()
									# String duplicates + id duplicates
									UpperCamelCase_		  =   sorted(vocab.keys()    )[:3]
									UpperCamelCase_		  =   [targets[0], targets[1], targets[0], targets[2], targets[1]]
									UpperCamelCase_		  =   fill_masker(F"""My name is {tokenizer.mask_token}"""     , targets=snake_case__     , top_k=10    )

									# The target list contains duplicates, so we can't output more
									# than them
									self.assertEqual(len(snake_case__    )     , 3    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   FillMaskPipeline(model=snake_case__     , tokenizer=snake_case__    )

									UpperCamelCase_		  =   fill_masker(
									    F"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""     , top_k=2    )
									self.assertEqual(
									    snake_case__     , [
									        [
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        ],
									        [
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        ],
									        [
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									            {"sequence": ANY(snake_case__    ), "score": ANY(snake_case__    ), "token": ANY(snake_case__    ), "token_str": ANY(snake_case__    )},
									        ],
									    ]     , )





from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel



def    _lowerCAmelCase    (_lowerCAmelCase   ,	_lowerCAmelCase   ,	_lowerCAmelCase=None   ,	_lowerCAmelCase=None):
					if attention_mask is None:
										UpperCamelCase_		  =   tf.cast(tf.math.not_equal(_lowerCAmelCase   ,	config.pad_token_id)   ,	tf.inta)
					return {"input_ids": input_ids, "attention_mask": attention_mask}


@require_tf
class       _lowercase   :
				'''simple docstring'''







				lowercase__   						=						OPTConfig
				lowercase__   						=						{}
				lowercase__   						=						"""gelu"""

				def __init__( self     , snake_case__     , snake_case__=13     , snake_case__=7     , snake_case__=True     , snake_case__=False     , snake_case__=99     , snake_case__=16     , snake_case__=2     , snake_case__=4     , snake_case__=4     , snake_case__="gelu"     , snake_case__=0.1     , snake_case__=0.1     , snake_case__=20     , snake_case__=2     , snake_case__=1     , snake_case__=0     , snake_case__=16     , snake_case__=16     , ):

									'''simple docstring'''


									UpperCamelCase_		  =   parent
									UpperCamelCase_		  =   batch_size
									UpperCamelCase_		  =   seq_length
									UpperCamelCase_		  =   is_training
									UpperCamelCase_		  =   use_labels
									UpperCamelCase_		  =   vocab_size
									UpperCamelCase_		  =   hidden_size
									UpperCamelCase_		  =   num_hidden_layers
									UpperCamelCase_		  =   num_attention_heads
									UpperCamelCase_		  =   intermediate_size
									UpperCamelCase_		  =   hidden_act
									UpperCamelCase_		  =   hidden_dropout_prob
									UpperCamelCase_		  =   attention_probs_dropout_prob
									UpperCamelCase_		  =   max_position_embeddings
									UpperCamelCase_		  =   eos_token_id
									UpperCamelCase_		  =   pad_token_id
									UpperCamelCase_		  =   bos_token_id
									UpperCamelCase_		  =   embed_dim
									UpperCamelCase_		  =   word_embed_proj_dim
									UpperCamelCase_		  =   False

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   ids_tensor([self.batch_size, self.seq_length - 1]     , self.vocab_size    )
									UpperCamelCase_		  =   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size    )     , 1    )
									UpperCamelCase_		  =   tf.concat([input_ids, eos_tensor]     , axis=1    )

									UpperCamelCase_		  =   self.config_cls(
									    vocab_size=self.vocab_size     , hidden_size=self.hidden_size     , num_hidden_layers=self.num_hidden_layers     , num_attention_heads=self.num_attention_heads     , ffn_dim=self.intermediate_size     , dropout=self.hidden_dropout_prob     , attention_dropout=self.attention_probs_dropout_prob     , max_position_embeddings=self.max_position_embeddings     , eos_token_id=self.eos_token_id     , bos_token_id=self.bos_token_id     , pad_token_id=self.pad_token_id     , embed_dim=self.embed_dim     , word_embed_proj_dim=self.word_embed_proj_dim     , is_encoder_decoder=snake_case__     , **self.config_updates     , )
									UpperCamelCase_		  =   prepare_opt_inputs_dict(snake_case__     , snake_case__    )
									return config, inputs_dict

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTModel(config=snake_case__    )
									UpperCamelCase_		  =   inputs_dict["input_ids"]

									UpperCamelCase_		  =   input_ids[:1, :]
									UpperCamelCase_		  =   inputs_dict["attention_mask"][:1, :]
									UpperCamelCase_		  =   1

									# first forward pass
									UpperCamelCase_		  =   model(snake_case__     , attention_mask=snake_case__     , use_cache=snake_case__    )

									UpperCamelCase_  ,    UpperCamelCase_		  =   outputs.to_tuple()

									# create hypothetical next token and extent to next_input_ids
									UpperCamelCase_		  =   ids_tensor((self.batch_size, 3)     , config.vocab_size    )
									UpperCamelCase_		  =   tf.cast(ids_tensor((self.batch_size, 3)     , 2    )     , tf.inta    )

									# append to next input_ids and
									UpperCamelCase_		  =   tf.concat([input_ids, next_tokens]     , axis=-1    )
									UpperCamelCase_		  =   tf.concat([attention_mask, next_attn_mask]     , axis=-1    )

									UpperCamelCase_		  =   model(snake_case__     , attention_mask=snake_case__    )[0]
									UpperCamelCase_		  =   model(snake_case__     , attention_mask=snake_case__     , past_key_values=snake_case__    )[0]

									self.parent.assertEqual(next_tokens.shape[1]     , output_from_past.shape[1]    )

									# select random slice
									UpperCamelCase_		  =   int(ids_tensor((1,)     , output_from_past.shape[-1]    )    )
									UpperCamelCase_		  =   output_from_no_past[:, -3:, random_slice_idx]
									UpperCamelCase_		  =   output_from_past[:, :, random_slice_idx]

									# test that outputs are equal for slice
									tf.debugging.assert_near(snake_case__     , snake_case__     , rtol=1e-3    )




@require_tf
class       _lowercase   (a_      ,   a_      ,   unittest.TestCase      ):
				'''simple docstring'''







				lowercase__   						=						(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
				lowercase__   						=						(TFOPTForCausalLM,) if is_tf_available() else ()
				lowercase__   						=						(
				    {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
				)
				lowercase__   						=						False
				lowercase__   						=						False
				lowercase__   						=						False
				lowercase__   						=						10

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTModelTester(self    )
									UpperCamelCase_		  =   ConfigTester(self     , config_class=snake_case__    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									self.config_tester.run_common_tests()

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_decoder_model_past_large_inputs(*snake_case__    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_  ,    UpperCamelCase_		  =   self.model_tester.prepare_config_and_inputs_for_common()

									def _get_word_embedding_weight(snake_case__     , snake_case__    ):
														if hasattr(snake_case__     , "weight"    ):
																			return embedding_layer.weight
														else:
																			# Here we build the word embeddings weights if not exists.
																			# And then we retry to get the attribute once built.
																			model.build()
																			if hasattr(snake_case__     , "weight"    ):
																								return embedding_layer.weight
																			else:
																								return None

									for model_class in self.all_model_classes:
														for size in [config.vocab_size - 10, config.vocab_size + 10]:
																			# build the embeddings
																			UpperCamelCase_		  =   model_class(config=snake_case__    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_input_embeddings()    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_output_embeddings()    )

																			# reshape the embeddings
																			model.resize_token_embeddings(snake_case__    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_input_embeddings()    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_output_embeddings()    )

																			# check that the resized embeddings size matches the desired size.
																			UpperCamelCase_		  =   size if size is not None else config.vocab_size

																			self.assertEqual(new_input_embeddings.shape[0]     , snake_case__    )

																			# check that weights remain the same after resizing
																			UpperCamelCase_		  =   True
																			for pa, pa in zip(old_input_embeddings.value()     , new_input_embeddings.value()    ):
																								if tf.math.reduce_sum(tf.math.abs(pa - pa    )    ) > 0:
																													UpperCamelCase_		  =   False
																			self.assertTrue(snake_case__    )

																			if old_output_embeddings is not None and new_output_embeddings is not None:
																								self.assertEqual(new_output_embeddings.shape[0]     , snake_case__    )

																								UpperCamelCase_		  =   True
																								for pa, pa in zip(old_output_embeddings.value()     , new_output_embeddings.value()    ):
																													if tf.math.reduce_sum(tf.math.abs(pa - pa    )    ) > 0:
																																		UpperCamelCase_		  =   False
																								self.assertTrue(snake_case__    )




def    _lowerCAmelCase    (_lowerCAmelCase):
					return tf.constant(_lowerCAmelCase   ,	dtype=tf.intaa)


@require_tf
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				lowercase__   						=						99

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   tf.ones((4, 1)     , dtype=tf.intaa    ) * 2
									UpperCamelCase_		  =   tf.concat([ids_tensor((4, 6)     , self.vocab_size - 3    ) + 3, eos_column_vector]     , axis=1    )
									UpperCamelCase_		  =   input_ids.shape[0]
									UpperCamelCase_		  =   OPTConfig(
									    vocab_size=self.vocab_size     , hidden_size=24     , num_hidden_layers=2     , num_attention_heads=2     , ffn_dim=32     , max_position_embeddings=48     , eos_token_id=2     , pad_token_id=1     , bos_token_id=0     , )
									return config, input_ids, batch_size




@require_sentencepiece
@require_tf
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				@slow
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTModel.from_pretrained("facebook/opt-350m"    )
									UpperCamelCase_		  =   _long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]    )
									UpperCamelCase_		  =   tf.not_equal(snake_case__     , model.config.pad_token_id    )
									with tf.GradientTape():
														UpperCamelCase_		  =   model(input_ids=snake_case__     , attention_mask=snake_case__    ).last_hidden_state
									UpperCamelCase_		  =   (1, 11, 512)
									self.assertEqual(output.shape     , snake_case__    )
									UpperCamelCase_		  =   tf.constant(
									    [[-0.2_873, -1.9_218, -0.3_033], [-1.2_710, -0.1_338, -0.1_902], [0.4_095, 0.1_214, -1.3_121]]    )
									self.assertTrue(np.allclose(output[:, :3, :3]     , snake_case__     , atol=4e-3    )    )

									UpperCamelCase_		  =   tf.function(snake_case__     , jit_compile=snake_case__    )
									UpperCamelCase_		  =   xla_generate(snake_case__     , snake_case__    )[0]
									self.assertTrue(np.allclose(output[:, :3, :3]     , snake_case__     , atol=4e-2    )    )




@require_tf
@slow
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									super().setUp()
									UpperCamelCase_		  =   "facebook/opt-350m"

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(self.path_model    )
									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(self.path_model    )

									UpperCamelCase_		  =   [
									    "Today is a beautiful day and I want to",
									    "In the city of",
									    "Paris is the capital of France and",
									    "Computers and mobile phones have taken",
									]
									# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
									UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"     , padding=snake_case__     , add_special_tokens=snake_case__    )
									UpperCamelCase_		  =   tf.math.reduce_mean(model(inputs.input_ids     , attention_mask=inputs.attention_mask    )[0]     , axis=-1    )
									UpperCamelCase_		  =   tf.constant(
									    [
									        [1.3_851, -13.8_923, -10.5_229, -10.7_533, -0.2_309, -10.2_384, -0.5_365, -9.0_947, -5.1_670],
									        [-4.7_073, -10.6_276, -3.9_415, -21.5_242, -0.2_822, -0.2_822, -0.2_822, -0.2_822, -0.2_822],
									        [0.6_247, -3.4_229, -8.9_179, -1.4_297, -14.1_650, 1.4_146, -9.0_218, -0.2_703, -0.2_703],
									        [6.4_783, -1.9_913, -10.7_926, -2.3_336, 1.5_092, -0.9_974, -6.8_213, 1.3_477, 1.3_477],
									    ]    )
									self.assertTrue(np.allclose(snake_case__     , snake_case__     , atol=1e-4    )    )

									UpperCamelCase_		  =   tf.function(snake_case__     , jit_compile=snake_case__    )
									UpperCamelCase_		  =   tf.math.reduce_mean(xla_generate(inputs.input_ids     , attention_mask=inputs.attention_mask    )[0]     , axis=-1    )
									self.assertTrue(np.allclose(snake_case__     , snake_case__     , atol=1e-4    )    )




@require_tf
@slow
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				@property
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									return [
									    "Today is a beautiful day and I want",
									    "In the city of",
									    "Paris is the capital of France and",
									    "Computers and mobile phones have taken",
									]

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   "facebook/opt-125m"

									UpperCamelCase_		  =   [
									    "Today is a beautiful day and I want to",
									    "In the city of New York, the city",
									    "Paris is the capital of France and the capital",
									    "Computers and mobile phones have taken over the",
									]

									UpperCamelCase_		  =   []
									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(snake_case__    )
									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(snake_case__    )

									for prompt in self.prompts:
														UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"    ).input_ids

														UpperCamelCase_		  =   model.generate(snake_case__     , max_length=10    )

														UpperCamelCase_		  =   tokenizer.batch_decode(snake_case__     , skip_special_tokens=snake_case__    )
														predicted_outputs += generated_string

									self.assertListEqual(snake_case__     , snake_case__    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   "facebook/opt-350m"

									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(snake_case__    )
									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(snake_case__    )

									UpperCamelCase_		  =   "left"

									# use different length sentences to test batching
									UpperCamelCase_		  =   [
									    "Hello, my dog is a little",
									    "Today, I",
									]

									UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"     , padding=snake_case__    )
									UpperCamelCase_		  =   inputs["input_ids"]

									UpperCamelCase_		  =   model.generate(input_ids=snake_case__     , attention_mask=inputs["attention_mask"]    )

									UpperCamelCase_		  =   tokenizer(sentences[0]     , return_tensors="tf"    ).input_ids
									UpperCamelCase_		  =   model.generate(input_ids=snake_case__    )

									UpperCamelCase_		  =   inputs_non_padded.shape[-1] - tf.math.reduce_sum(
									    tf.cast(inputs["attention_mask"][-1]     , tf.intaa    )    )
									UpperCamelCase_		  =   tokenizer(sentences[1]     , return_tensors="tf"    ).input_ids
									UpperCamelCase_		  =   model.generate(input_ids=snake_case__     , max_length=model.config.max_length - num_paddings    )

									UpperCamelCase_		  =   tokenizer.batch_decode(snake_case__     , skip_special_tokens=snake_case__    )
									UpperCamelCase_		  =   tokenizer.decode(output_non_padded[0]     , skip_special_tokens=snake_case__    )
									UpperCamelCase_		  =   tokenizer.decode(output_padded[0]     , skip_special_tokens=snake_case__    )

									UpperCamelCase_		  =   [
									    "Hello, my dog is a little bit of a dork.\nI'm a little bit",
									    "Today, I was in the middle of a conversation with a friend about the",
									]
									self.assertListEqual(snake_case__     , snake_case__    )
									self.assertListEqual(snake_case__     , [non_padded_sentence, padded_sentence]    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   "facebook/opt-350m"

									UpperCamelCase_		  =   [
									    "Today is a beautiful day and I want to",
									    "In the city of San Francisco, the city",
									    "Paris is the capital of France and the capital",
									    "Computers and mobile phones have taken over the",
									]

									UpperCamelCase_		  =   []
									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(snake_case__    )
									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(snake_case__    )

									for prompt in self.prompts:
														UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"    ).input_ids

														UpperCamelCase_		  =   model.generate(snake_case__     , max_length=10    )

														UpperCamelCase_		  =   tokenizer.batch_decode(snake_case__     , skip_special_tokens=snake_case__    )
														predicted_outputs += generated_string

									self.assertListEqual(snake_case__     , snake_case__    )
