



from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel







def snake_case__  (			__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=None			,     __SCREAMING_SNAKE_CASE=None )		->							List[str]:
		if attention_mask is None:
				UpperCAmelCase_						=				tf.cast(tf.math.not_equal(__SCREAMING_SNAKE_CASE			,     config.pad_token_id )			,     tf.inta )
		return {"input_ids": input_ids, "attention_mask": attention_mask}



@require_tf
class 	lowerCamelCase      :







					'''simple docstring'''







					lowerCAmelCase_				:    Any									=  OPTConfig
					lowerCAmelCase_				:    int									=  {}
					lowerCAmelCase_				:    Union[str, Any]									=  'gelu'



					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase=13    ,	lowerCAmelCase=7    ,	lowerCAmelCase=True    ,	lowerCAmelCase=False    ,	lowerCAmelCase=99    ,	lowerCAmelCase=16    ,	lowerCAmelCase=2    ,	lowerCAmelCase=4    ,	lowerCAmelCase=4    ,	lowerCAmelCase="gelu"    ,	lowerCAmelCase=0.1    ,	lowerCAmelCase=0.1    ,	lowerCAmelCase=20    ,	lowerCAmelCase=2    ,	lowerCAmelCase=1    ,	lowerCAmelCase=0    ,	lowerCAmelCase=16    ,	lowerCAmelCase=16    ,	):
							UpperCAmelCase_						=				parent
							UpperCAmelCase_						=				batch_size
							UpperCAmelCase_						=				seq_length
							UpperCAmelCase_						=				is_training
							UpperCAmelCase_						=				use_labels
							UpperCAmelCase_						=				vocab_size
							UpperCAmelCase_						=				hidden_size
							UpperCAmelCase_						=				num_hidden_layers
							UpperCAmelCase_						=				num_attention_heads
							UpperCAmelCase_						=				intermediate_size
							UpperCAmelCase_						=				hidden_act
							UpperCAmelCase_						=				hidden_dropout_prob
							UpperCAmelCase_						=				attention_probs_dropout_prob
							UpperCAmelCase_						=				max_position_embeddings
							UpperCAmelCase_						=				eos_token_id
							UpperCAmelCase_						=				pad_token_id
							UpperCAmelCase_						=				bos_token_id
							UpperCAmelCase_						=				embed_dim
							UpperCAmelCase_						=				word_embed_proj_dim
							UpperCAmelCase_						=				False



					def    A__		(			self     ):
							UpperCAmelCase_						=				ids_tensor([self.batch_size, self.seq_length - 1]    ,	self.vocab_size     )
							UpperCAmelCase_						=				tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size     )    ,	1     )
							UpperCAmelCase_						=				tf.concat([input_ids, eos_tensor]    ,	axis=1     )

							UpperCAmelCase_						=				self.config_cls(
							    vocab_size=self.vocab_size    ,	hidden_size=self.hidden_size    ,	num_hidden_layers=self.num_hidden_layers    ,	num_attention_heads=self.num_attention_heads    ,	ffn_dim=self.intermediate_size    ,	dropout=self.hidden_dropout_prob    ,	attention_dropout=self.attention_probs_dropout_prob    ,	max_position_embeddings=self.max_position_embeddings    ,	eos_token_id=self.eos_token_id    ,	bos_token_id=self.bos_token_id    ,	pad_token_id=self.pad_token_id    ,	embed_dim=self.embed_dim    ,	word_embed_proj_dim=self.word_embed_proj_dim    ,	is_encoder_decoder=lowerCAmelCase    ,	**self.config_updates    ,	)
							UpperCAmelCase_						=				prepare_opt_inputs_dict(lowerCAmelCase    ,	lowerCAmelCase     )
							return config, inputs_dict



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							UpperCAmelCase_						=				TFOPTModel(config=lowerCAmelCase     )
							UpperCAmelCase_						=				inputs_dict["input_ids"]

							UpperCAmelCase_						=				input_ids[:1, :]
							UpperCAmelCase_						=				inputs_dict["attention_mask"][:1, :]
							UpperCAmelCase_						=				1

							# first forward pass
							UpperCAmelCase_						=				model(lowerCAmelCase    ,	attention_mask=lowerCAmelCase    ,	use_cache=lowerCAmelCase     )

							UpperCAmelCase_					,						UpperCAmelCase_						=				outputs.to_tuple()

							# create hypothetical next token and extent to next_input_ids
							UpperCAmelCase_						=				ids_tensor((self.batch_size, 3)    ,	config.vocab_size     )
							UpperCAmelCase_						=				tf.cast(ids_tensor((self.batch_size, 3)    ,	2     )    ,	tf.inta     )

							# append to next input_ids and
							UpperCAmelCase_						=				tf.concat([input_ids, next_tokens]    ,	axis=-1     )
							UpperCAmelCase_						=				tf.concat([attention_mask, next_attn_mask]    ,	axis=-1     )

							UpperCAmelCase_						=				model(lowerCAmelCase    ,	attention_mask=lowerCAmelCase     )[0]
							UpperCAmelCase_						=				model(lowerCAmelCase    ,	attention_mask=lowerCAmelCase    ,	past_key_values=lowerCAmelCase     )[0]

							self.parent.assertEqual(next_tokens.shape[1]    ,	output_from_past.shape[1]     )

							# select random slice
							UpperCAmelCase_						=				int(ids_tensor((1,)    ,	output_from_past.shape[-1]     )     )
							UpperCAmelCase_						=				output_from_no_past[:, -3:, random_slice_idx]
							UpperCAmelCase_						=				output_from_past[:, :, random_slice_idx]

							# test that outputs are equal for slice
							tf.debugging.assert_near(lowerCAmelCase    ,	lowerCAmelCase    ,	rtol=1e-3     )





@require_tf
class 	lowerCamelCase      (						lowercase__,     lowercase__,     unittest.TestCase  ):







					'''simple docstring'''







					lowerCAmelCase_				:    Tuple									=  (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
					lowerCAmelCase_				:    Optional[int]									=  (TFOPTForCausalLM,) if is_tf_available() else ()
					lowerCAmelCase_				:    Tuple									=  (
					    {'feature-extraction': TFOPTModel, 'text-generation': TFOPTForCausalLM} if is_tf_available() else {}
					)
					lowerCAmelCase_				:    List[str]									=  False
					lowerCAmelCase_				:    Tuple									=  False
					lowerCAmelCase_				:    Union[str, Any]									=  False
					lowerCAmelCase_				:    Tuple									=  10



					def    A__		(			self     ):
							UpperCAmelCase_						=				TFOPTModelTester(self     )
							UpperCAmelCase_						=				ConfigTester(self    ,	config_class=lowerCAmelCase     )



					def    A__		(			self     ):
							self.config_tester.run_common_tests()



					def    A__		(			self     ):
							UpperCAmelCase_						=				self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase     )



					def    A__		(			self     ):
							UpperCAmelCase_					,						UpperCAmelCase_						=				self.model_tester.prepare_config_and_inputs_for_common()

							def _get_word_embedding_weight(lowerCAmelCase    ,	lowerCAmelCase     ):
									if hasattr(lowerCAmelCase    ,	"weight"     ):
											return embedding_layer.weight
									else:
											# Here we build the word embeddings weights if not exists.
											# And then we retry to get the attribute once built.
											model.build()
											if hasattr(lowerCAmelCase    ,	"weight"     ):
													return embedding_layer.weight
											else:
													return None

							for model_class in self.all_model_classes:
									for size in [config.vocab_size - 10, config.vocab_size + 10]:
											# build the embeddings
											UpperCAmelCase_						=				model_class(config=lowerCAmelCase     )
											UpperCAmelCase_						=				_get_word_embedding_weight(lowerCAmelCase    ,	model.get_input_embeddings()     )
											UpperCAmelCase_						=				_get_word_embedding_weight(lowerCAmelCase    ,	model.get_output_embeddings()     )

											# reshape the embeddings
											model.resize_token_embeddings(lowerCAmelCase     )
											UpperCAmelCase_						=				_get_word_embedding_weight(lowerCAmelCase    ,	model.get_input_embeddings()     )
											UpperCAmelCase_						=				_get_word_embedding_weight(lowerCAmelCase    ,	model.get_output_embeddings()     )

											# check that the resized embeddings size matches the desired size.
											UpperCAmelCase_						=				size if size is not None else config.vocab_size

											self.assertEqual(new_input_embeddings.shape[0]    ,	lowerCAmelCase     )

											# check that weights remain the same after resizing
											UpperCAmelCase_						=				True
											for pa, pa in zip(old_input_embeddings.value()    ,	new_input_embeddings.value()     ):
													if tf.math.reduce_sum(tf.math.abs(pa - pa     )     ) > 0:
															UpperCAmelCase_						=				False
											self.assertTrue(lowerCAmelCase     )

											if old_output_embeddings is not None and new_output_embeddings is not None:
													self.assertEqual(new_output_embeddings.shape[0]    ,	lowerCAmelCase     )

													UpperCAmelCase_						=				True
													for pa, pa in zip(old_output_embeddings.value()    ,	new_output_embeddings.value()     ):
															if tf.math.reduce_sum(tf.math.abs(pa - pa     )     ) > 0:
																	UpperCAmelCase_						=				False
													self.assertTrue(lowerCAmelCase     )







def snake_case__  (			__SCREAMING_SNAKE_CASE )		->							Optional[int]:
		return tf.constant(__SCREAMING_SNAKE_CASE			,     dtype=tf.intaa )



@require_tf
class 	lowerCamelCase      (						unittest.TestCase  ):







					'''simple docstring'''







					lowerCAmelCase_				:    List[Any]									=  99



					def    A__		(			self     ):
							UpperCAmelCase_						=				tf.ones((4, 1)    ,	dtype=tf.intaa     ) * 2
							UpperCAmelCase_						=				tf.concat([ids_tensor((4, 6)    ,	self.vocab_size - 3     ) + 3, eos_column_vector]    ,	axis=1     )
							UpperCAmelCase_						=				input_ids.shape[0]
							UpperCAmelCase_						=				OPTConfig(
							    vocab_size=self.vocab_size    ,	hidden_size=24    ,	num_hidden_layers=2    ,	num_attention_heads=2    ,	ffn_dim=32    ,	max_position_embeddings=48    ,	eos_token_id=2    ,	pad_token_id=1    ,	bos_token_id=0    ,	)
							return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class 	lowerCamelCase      (						unittest.TestCase  ):







					'''simple docstring'''







					@slow
					def    A__		(			self     ):
							UpperCAmelCase_						=				TFOPTModel.from_pretrained("facebook/opt-350m"     )
							UpperCAmelCase_						=				_long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]     )
							UpperCAmelCase_						=				tf.not_equal(lowerCAmelCase    ,	model.config.pad_token_id     )
							with tf.GradientTape():
									UpperCAmelCase_						=				model(input_ids=lowerCAmelCase    ,	attention_mask=lowerCAmelCase     ).last_hidden_state
							UpperCAmelCase_						=				(1, 11, 512)
							self.assertEqual(output.shape    ,	lowerCAmelCase     )
							UpperCAmelCase_						=				tf.constant(
							    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]     )
							self.assertTrue(np.allclose(output[:, :3, :3]    ,	lowerCAmelCase    ,	atol=4e-3     )     )

							UpperCAmelCase_						=				tf.function(lowerCAmelCase    ,	jit_compile=lowerCAmelCase     )
							UpperCAmelCase_						=				xla_generate(lowerCAmelCase    ,	lowerCAmelCase     )[0]
							self.assertTrue(np.allclose(output[:, :3, :3]    ,	lowerCAmelCase    ,	atol=4e-2     )     )





@require_tf
@slow
class 	lowerCamelCase      (						unittest.TestCase  ):







					'''simple docstring'''







					def    A__		(			self     ):
							super().setUp()
							UpperCAmelCase_						=				"facebook/opt-350m"



					def    A__		(			self     ):
							UpperCAmelCase_						=				TFOPTForCausalLM.from_pretrained(self.path_model     )
							UpperCAmelCase_						=				GPTaTokenizer.from_pretrained(self.path_model     )

							UpperCAmelCase_						=				[
							    "Today is a beautiful day and I want to",
							    "In the city of",
							    "Paris is the capital of France and",
							    "Computers and mobile phones have taken",
							]
							# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
							UpperCAmelCase_						=				tokenizer(lowerCAmelCase    ,	return_tensors="tf"    ,	padding=lowerCAmelCase    ,	add_special_tokens=lowerCAmelCase     )
							UpperCAmelCase_						=				tf.math.reduce_mean(model(inputs.input_ids    ,	attention_mask=inputs.attention_mask     )[0]    ,	axis=-1     )
							UpperCAmelCase_						=				tf.constant(
							    [
							        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
							        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
							        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
							        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
							    ]     )
							self.assertTrue(np.allclose(lowerCAmelCase    ,	lowerCAmelCase    ,	atol=1e-4     )     )

							UpperCAmelCase_						=				tf.function(lowerCAmelCase    ,	jit_compile=lowerCAmelCase     )
							UpperCAmelCase_						=				tf.math.reduce_mean(xla_generate(inputs.input_ids    ,	attention_mask=inputs.attention_mask     )[0]    ,	axis=-1     )
							self.assertTrue(np.allclose(lowerCAmelCase    ,	lowerCAmelCase    ,	atol=1e-4     )     )





@require_tf
@slow
class 	lowerCamelCase      (						unittest.TestCase  ):







					'''simple docstring'''







					@property
					def    A__		(			self     ):
							return [
							    "Today is a beautiful day and I want",
							    "In the city of",
							    "Paris is the capital of France and",
							    "Computers and mobile phones have taken",
							]



					def    A__		(			self     ):
							UpperCAmelCase_						=				"facebook/opt-125m"

							UpperCAmelCase_						=				[
							    "Today is a beautiful day and I want to",
							    "In the city of New York, the city",
							    "Paris is the capital of France and the capital",
							    "Computers and mobile phones have taken over the",
							]

							UpperCAmelCase_						=				[]
							UpperCAmelCase_						=				GPTaTokenizer.from_pretrained(lowerCAmelCase     )
							UpperCAmelCase_						=				TFOPTForCausalLM.from_pretrained(lowerCAmelCase     )

							for prompt in self.prompts:
									UpperCAmelCase_						=				tokenizer(lowerCAmelCase    ,	return_tensors="tf"     ).input_ids

									UpperCAmelCase_						=				model.generate(lowerCAmelCase    ,	max_length=10     )

									UpperCAmelCase_						=				tokenizer.batch_decode(lowerCAmelCase    ,	skip_special_tokens=lowerCAmelCase     )
									predicted_outputs += generated_string

							self.assertListEqual(lowerCAmelCase    ,	lowerCAmelCase     )



					def    A__		(			self     ):
							UpperCAmelCase_						=				"facebook/opt-350m"

							UpperCAmelCase_						=				GPTaTokenizer.from_pretrained(lowerCAmelCase     )
							UpperCAmelCase_						=				TFOPTForCausalLM.from_pretrained(lowerCAmelCase     )

							UpperCAmelCase_						=				"left"

							# use different length sentences to test batching
							UpperCAmelCase_						=				[
							    "Hello, my dog is a little",
							    "Today, I",
							]

							UpperCAmelCase_						=				tokenizer(lowerCAmelCase    ,	return_tensors="tf"    ,	padding=lowerCAmelCase     )
							UpperCAmelCase_						=				inputs["input_ids"]

							UpperCAmelCase_						=				model.generate(input_ids=lowerCAmelCase    ,	attention_mask=inputs["attention_mask"]     )

							UpperCAmelCase_						=				tokenizer(sentences[0]    ,	return_tensors="tf"     ).input_ids
							UpperCAmelCase_						=				model.generate(input_ids=lowerCAmelCase     )

							UpperCAmelCase_						=				inputs_non_padded.shape[-1] - tf.math.reduce_sum(
							    tf.cast(inputs["attention_mask"][-1]    ,	tf.intaa     )     )
							UpperCAmelCase_						=				tokenizer(sentences[1]    ,	return_tensors="tf"     ).input_ids
							UpperCAmelCase_						=				model.generate(input_ids=lowerCAmelCase    ,	max_length=model.config.max_length - num_paddings     )

							UpperCAmelCase_						=				tokenizer.batch_decode(lowerCAmelCase    ,	skip_special_tokens=lowerCAmelCase     )
							UpperCAmelCase_						=				tokenizer.decode(output_non_padded[0]    ,	skip_special_tokens=lowerCAmelCase     )
							UpperCAmelCase_						=				tokenizer.decode(output_padded[0]    ,	skip_special_tokens=lowerCAmelCase     )

							UpperCAmelCase_						=				[
							    "Hello, my dog is a little bit of a dork.\nI'm a little bit",
							    "Today, I was in the middle of a conversation with a friend about the",
							]
							self.assertListEqual(lowerCAmelCase    ,	lowerCAmelCase     )
							self.assertListEqual(lowerCAmelCase    ,	[non_padded_sentence, padded_sentence]     )



					def    A__		(			self     ):
							UpperCAmelCase_						=				"facebook/opt-350m"

							UpperCAmelCase_						=				[
							    "Today is a beautiful day and I want to",
							    "In the city of San Francisco, the city",
							    "Paris is the capital of France and the capital",
							    "Computers and mobile phones have taken over the",
							]

							UpperCAmelCase_						=				[]
							UpperCAmelCase_						=				GPTaTokenizer.from_pretrained(lowerCAmelCase     )
							UpperCAmelCase_						=				TFOPTForCausalLM.from_pretrained(lowerCAmelCase     )

							for prompt in self.prompts:
									UpperCAmelCase_						=				tokenizer(lowerCAmelCase    ,	return_tensors="tf"     ).input_ids

									UpperCAmelCase_						=				model.generate(lowerCAmelCase    ,	max_length=10     )

									UpperCAmelCase_						=				tokenizer.batch_decode(lowerCAmelCase    ,	skip_special_tokens=lowerCAmelCase     )
									predicted_outputs += generated_string

							self.assertListEqual(lowerCAmelCase    ,	lowerCAmelCase     )






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE     				=						{"configuration_xlnet": ["XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLNetConfig"]}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						["XLNetTokenizer"]

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						["XLNetTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						[
			    "XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "XLNetForMultipleChoice",
			    "XLNetForQuestionAnswering",
			    "XLNetForQuestionAnsweringSimple",
			    "XLNetForSequenceClassification",
			    "XLNetForTokenClassification",
			    "XLNetLMHeadModel",
			    "XLNetModel",
			    "XLNetPreTrainedModel",
			    "load_tf_weights_in_xlnet",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						[
			    "TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFXLNetForMultipleChoice",
			    "TFXLNetForQuestionAnsweringSimple",
			    "TFXLNetForSequenceClassification",
			    "TFXLNetForTokenClassification",
			    "TFXLNetLMHeadModel",
			    "TFXLNetMainLayer",
			    "TFXLNetModel",
			    "TFXLNetPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_xlnet import XLNetTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_xlnet_fast import XLNetTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_xlnet import (
						    XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
						    XLNetForMultipleChoice,
						    XLNetForQuestionAnswering,
						    XLNetForQuestionAnsweringSimple,
						    XLNetForSequenceClassification,
						    XLNetForTokenClassification,
						    XLNetLMHeadModel,
						    XLNetModel,
						    XLNetPreTrainedModel,
						    load_tf_weights_in_xlnet,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_xlnet import (
						    TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFXLNetForMultipleChoice,
						    TFXLNetForQuestionAnsweringSimple,
						    TFXLNetForSequenceClassification,
						    TFXLNetForTokenClassification,
						    TFXLNetLMHeadModel,
						    TFXLNetMainLayer,
						    TFXLNetModel,
						    TFXLNetPreTrainedModel,
						)

else:
			import sys

			SCREAMING_SNAKE_CASE     				=						_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

