




from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE=None					,	SCREAMING_SNAKE_CASE=None   ):
			'''simple docstring'''
			if attention_mask is None:
						SCREAMING_SNAKE_CASE           =					tf.cast(tf.math.not_equal(SCREAMING_SNAKE_CASE					,	config.pad_token_id   )					,	tf.inta   )
			return {"input_ids": input_ids, "attention_mask": attention_mask}




@require_tf
class 			UpperCAmelCase_      :




							'''simple docstring'''



							a__						  =     OPTConfig
							a__						  =     {}
							a__						  =     '''gelu'''





							def __init__(					self       :	List[Any]       ,     a       :	Optional[int]       ,     a       :	List[str]=13       ,     a       :	Optional[int]=7       ,     a       :	str=True       ,     a       :	Any=False       ,     a       :	Tuple=99       ,     a       :	int=16       ,     a       :	Tuple=2       ,     a       :	List[str]=4       ,     a       :	Optional[int]=4       ,     a       :	int="gelu"       ,     a       :	Union[str, Any]=0.1       ,     a       :	int=0.1       ,     a       :	Optional[int]=20       ,     a       :	Any=2       ,     a       :	Optional[Any]=1       ,     a       :	Optional[Any]=0       ,     a       :	Union[str, Any]=16       ,     a       :	Optional[Any]=16       ,     )			->					int:
										SCREAMING_SNAKE_CASE           =					parent
										SCREAMING_SNAKE_CASE           =					batch_size
										SCREAMING_SNAKE_CASE           =					seq_length
										SCREAMING_SNAKE_CASE           =					is_training
										SCREAMING_SNAKE_CASE           =					use_labels
										SCREAMING_SNAKE_CASE           =					vocab_size
										SCREAMING_SNAKE_CASE           =					hidden_size
										SCREAMING_SNAKE_CASE           =					num_hidden_layers
										SCREAMING_SNAKE_CASE           =					num_attention_heads
										SCREAMING_SNAKE_CASE           =					intermediate_size
										SCREAMING_SNAKE_CASE           =					hidden_act
										SCREAMING_SNAKE_CASE           =					hidden_dropout_prob
										SCREAMING_SNAKE_CASE           =					attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE           =					max_position_embeddings
										SCREAMING_SNAKE_CASE           =					eos_token_id
										SCREAMING_SNAKE_CASE           =					pad_token_id
										SCREAMING_SNAKE_CASE           =					bos_token_id
										SCREAMING_SNAKE_CASE           =					embed_dim
										SCREAMING_SNAKE_CASE           =					word_embed_proj_dim
										SCREAMING_SNAKE_CASE           =					False





							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Dict:
										SCREAMING_SNAKE_CASE           =					ids_tensor([self.batch_size, self.seq_length - 1]       ,     self.vocab_size			)
										SCREAMING_SNAKE_CASE           =					tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size			)       ,     1			)
										SCREAMING_SNAKE_CASE           =					tf.concat([input_ids, eos_tensor]       ,     axis=1			)

										SCREAMING_SNAKE_CASE           =					self.config_cls(
										    vocab_size=self.vocab_size       ,     hidden_size=self.hidden_size       ,     num_hidden_layers=self.num_hidden_layers       ,     num_attention_heads=self.num_attention_heads       ,     ffn_dim=self.intermediate_size       ,     dropout=self.hidden_dropout_prob       ,     attention_dropout=self.attention_probs_dropout_prob       ,     max_position_embeddings=self.max_position_embeddings       ,     eos_token_id=self.eos_token_id       ,     bos_token_id=self.bos_token_id       ,     pad_token_id=self.pad_token_id       ,     embed_dim=self.embed_dim       ,     word_embed_proj_dim=self.word_embed_proj_dim       ,     is_encoder_decoder=a       ,     **self.config_updates       ,     )
										SCREAMING_SNAKE_CASE           =					prepare_opt_inputs_dict(a       ,     a			)
										return config, inputs_dict







							def    _UpperCAmelCase     (					self       :	List[Any]       ,     a       :	Optional[Any]       ,     a       :	str			)			->					str:
										SCREAMING_SNAKE_CASE           =					TFOPTModel(config=a			)
										SCREAMING_SNAKE_CASE           =					inputs_dict["""input_ids"""]

										SCREAMING_SNAKE_CASE           =					input_ids[:1, :]
										SCREAMING_SNAKE_CASE           =					inputs_dict["""attention_mask"""][:1, :]
										SCREAMING_SNAKE_CASE           =					1

										# first forward pass
										SCREAMING_SNAKE_CASE           =					model(a       ,     attention_mask=a       ,     use_cache=a			)

										SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					outputs.to_tuple()

										# create hypothetical next token and extent to next_input_ids
										SCREAMING_SNAKE_CASE           =					ids_tensor((self.batch_size, 3)       ,     config.vocab_size			)
										SCREAMING_SNAKE_CASE           =					tf.cast(ids_tensor((self.batch_size, 3)       ,     2			)       ,     tf.inta			)

										# append to next input_ids and
										SCREAMING_SNAKE_CASE           =					tf.concat([input_ids, next_tokens]       ,     axis=-1			)
										SCREAMING_SNAKE_CASE           =					tf.concat([attention_mask, next_attn_mask]       ,     axis=-1			)

										SCREAMING_SNAKE_CASE           =					model(a       ,     attention_mask=a			)[0]
										SCREAMING_SNAKE_CASE           =					model(a       ,     attention_mask=a       ,     past_key_values=a			)[0]

										self.parent.assertEqual(next_tokens.shape[1]       ,     output_from_past.shape[1]			)

										# select random slice
										SCREAMING_SNAKE_CASE           =					int(ids_tensor((1,)       ,     output_from_past.shape[-1]			)			)
										SCREAMING_SNAKE_CASE           =					output_from_no_past[:, -3:, random_slice_idx]
										SCREAMING_SNAKE_CASE           =					output_from_past[:, :, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(a       ,     a       ,     rtol=1E-3			)





@require_tf
class 			UpperCAmelCase_      (   A		,      A		,      unittest.TestCase       ):




							'''simple docstring'''



							a__						  =     (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
							a__						  =     (TFOPTForCausalLM,) if is_tf_available() else ()
							a__						  =     (
							    {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
							)
							a__						  =     False
							a__						  =     False
							a__						  =     False
							a__						  =     1_0





							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Tuple:
										SCREAMING_SNAKE_CASE           =					TFOPTModelTester(self			)
										SCREAMING_SNAKE_CASE           =					ConfigTester(self       ,     config_class=a			)





							def    _UpperCAmelCase     (					self       :	Optional[Any]			)			->					List[Any]:
										self.config_tester.run_common_tests()





							def    _UpperCAmelCase     (					self       :	List[str]			)			->					Any:
										SCREAMING_SNAKE_CASE           =					self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.check_decoder_model_past_large_inputs(*a			)







							def    _UpperCAmelCase     (					self       :	Any			)			->					int:
										SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					self.model_tester.prepare_config_and_inputs_for_common()

										def _get_word_embedding_weight(a       :	int       ,     a       :	List[str]			):
													if hasattr(a       ,     """weight"""			):
																return embedding_layer.weight
													else:
																# Here we build the word embeddings weights if not exists.
																# And then we retry to get the attribute once built.
																model.build()
																if hasattr(a       ,     """weight"""			):
																			return embedding_layer.weight
																else:
																			return None

										for model_class in self.all_model_classes:
													for size in [config.vocab_size - 10, config.vocab_size + 10]:
																# build the embeddings
																SCREAMING_SNAKE_CASE           =					model_class(config=a			)
																SCREAMING_SNAKE_CASE           =					_get_word_embedding_weight(a       ,     model.get_input_embeddings()			)
																SCREAMING_SNAKE_CASE           =					_get_word_embedding_weight(a       ,     model.get_output_embeddings()			)

																# reshape the embeddings
																model.resize_token_embeddings(a			)
																SCREAMING_SNAKE_CASE           =					_get_word_embedding_weight(a       ,     model.get_input_embeddings()			)
																SCREAMING_SNAKE_CASE           =					_get_word_embedding_weight(a       ,     model.get_output_embeddings()			)

																# check that the resized embeddings size matches the desired size.
																SCREAMING_SNAKE_CASE           =					size if size is not None else config.vocab_size

																self.assertEqual(new_input_embeddings.shape[0]       ,     a			)

																# check that weights remain the same after resizing
																SCREAMING_SNAKE_CASE           =					True
																for pa, pa in zip(old_input_embeddings.value()       ,     new_input_embeddings.value()			):
																			if tf.math.reduce_sum(tf.math.abs(pa - pa			)			) > 0:
																						SCREAMING_SNAKE_CASE           =					False
																self.assertTrue(a			)

																if old_output_embeddings is not None and new_output_embeddings is not None:
																			self.assertEqual(new_output_embeddings.shape[0]       ,     a			)

																			SCREAMING_SNAKE_CASE           =					True
																			for pa, pa in zip(old_output_embeddings.value()       ,     new_output_embeddings.value()			):
																						if tf.math.reduce_sum(tf.math.abs(pa - pa			)			) > 0:
																									SCREAMING_SNAKE_CASE           =					False
																			self.assertTrue(a			)





def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			return tf.constant(SCREAMING_SNAKE_CASE					,	dtype=tf.intaa   )




@require_tf
class 			UpperCAmelCase_      (   unittest.TestCase       ):




							'''simple docstring'''



							a__						  =     9_9





							def    _UpperCAmelCase     (					self       :	Any			)			->					Union[str, Any]:
										SCREAMING_SNAKE_CASE           =					tf.ones((4, 1)       ,     dtype=tf.intaa			) * 2
										SCREAMING_SNAKE_CASE           =					tf.concat([ids_tensor((4, 6)       ,     self.vocab_size - 3			) + 3, eos_column_vector]       ,     axis=1			)
										SCREAMING_SNAKE_CASE           =					input_ids.shape[0]
										SCREAMING_SNAKE_CASE           =					OPTConfig(
										    vocab_size=self.vocab_size       ,     hidden_size=24       ,     num_hidden_layers=2       ,     num_attention_heads=2       ,     ffn_dim=32       ,     max_position_embeddings=48       ,     eos_token_id=2       ,     pad_token_id=1       ,     bos_token_id=0       ,     )
										return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class 			UpperCAmelCase_      (   unittest.TestCase       ):




							'''simple docstring'''



							@slow
							def    _UpperCAmelCase     (					self       :	List[str]			)			->					Optional[int]:
										SCREAMING_SNAKE_CASE           =					TFOPTModel.from_pretrained("""facebook/opt-350m"""			)
										SCREAMING_SNAKE_CASE           =					_long_tensor([[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]]			)
										SCREAMING_SNAKE_CASE           =					tf.not_equal(a       ,     model.config.pad_token_id			)
										with tf.GradientTape():
													SCREAMING_SNAKE_CASE           =					model(input_ids=a       ,     attention_mask=a			).last_hidden_state
										SCREAMING_SNAKE_CASE           =					(1, 11, 512)
										self.assertEqual(output.shape       ,     a			)
										SCREAMING_SNAKE_CASE           =					tf.constant(
										    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]			)
										self.assertTrue(np.allclose(output[:, :3, :3]       ,     a       ,     atol=4E-3			)			)

										SCREAMING_SNAKE_CASE           =					tf.function(a       ,     jit_compile=a			)
										SCREAMING_SNAKE_CASE           =					xla_generate(a       ,     a			)[0]
										self.assertTrue(np.allclose(output[:, :3, :3]       ,     a       ,     atol=4E-2			)			)





@require_tf
@slow
class 			UpperCAmelCase_      (   unittest.TestCase       ):




							'''simple docstring'''



							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										super().setUp()
										SCREAMING_SNAKE_CASE           =					"""facebook/opt-350m"""







							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					Any:
										SCREAMING_SNAKE_CASE           =					TFOPTForCausalLM.from_pretrained(self.path_model			)
										SCREAMING_SNAKE_CASE           =					GPTaTokenizer.from_pretrained(self.path_model			)

										SCREAMING_SNAKE_CASE           =					[
										    """Today is a beautiful day and I want to""",
										    """In the city of""",
										    """Paris is the capital of France and""",
										    """Computers and mobile phones have taken""",
										]
										# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
										SCREAMING_SNAKE_CASE           =					tokenizer(a       ,     return_tensors="""tf"""       ,     padding=a       ,     add_special_tokens=a			)
										SCREAMING_SNAKE_CASE           =					tf.math.reduce_mean(model(inputs.input_ids       ,     attention_mask=inputs.attention_mask			)[0]       ,     axis=-1			)
										SCREAMING_SNAKE_CASE           =					tf.constant(
										    [
										        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
										        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
										        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
										        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
										    ]			)
										self.assertTrue(np.allclose(a       ,     a       ,     atol=1E-4			)			)

										SCREAMING_SNAKE_CASE           =					tf.function(a       ,     jit_compile=a			)
										SCREAMING_SNAKE_CASE           =					tf.math.reduce_mean(xla_generate(inputs.input_ids       ,     attention_mask=inputs.attention_mask			)[0]       ,     axis=-1			)
										self.assertTrue(np.allclose(a       ,     a       ,     atol=1E-4			)			)





@require_tf
@slow
class 			UpperCAmelCase_      (   unittest.TestCase       ):




							'''simple docstring'''



							@property
							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										return [
										    "Today is a beautiful day and I want",
										    "In the city of",
										    "Paris is the capital of France and",
										    "Computers and mobile phones have taken",
										]





							def    _UpperCAmelCase     (					self       :	int			)			->					int:
										SCREAMING_SNAKE_CASE           =					"""facebook/opt-125m"""

										SCREAMING_SNAKE_CASE           =					[
										    """Today is a beautiful day and I want to""",
										    """In the city of New York, the city""",
										    """Paris is the capital of France and the capital""",
										    """Computers and mobile phones have taken over the""",
										]

										SCREAMING_SNAKE_CASE           =					[]
										SCREAMING_SNAKE_CASE           =					GPTaTokenizer.from_pretrained(a			)
										SCREAMING_SNAKE_CASE           =					TFOPTForCausalLM.from_pretrained(a			)

										for prompt in self.prompts:
													SCREAMING_SNAKE_CASE           =					tokenizer(a       ,     return_tensors="""tf"""			).input_ids

													SCREAMING_SNAKE_CASE           =					model.generate(a       ,     max_length=10			)

													SCREAMING_SNAKE_CASE           =					tokenizer.batch_decode(a       ,     skip_special_tokens=a			)
													predicted_outputs += generated_string

										self.assertListEqual(a       ,     a			)





							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										SCREAMING_SNAKE_CASE           =					"""facebook/opt-350m"""

										SCREAMING_SNAKE_CASE           =					GPTaTokenizer.from_pretrained(a			)
										SCREAMING_SNAKE_CASE           =					TFOPTForCausalLM.from_pretrained(a			)

										SCREAMING_SNAKE_CASE           =					"""left"""

										# use different length sentences to test batching
										SCREAMING_SNAKE_CASE           =					[
										    """Hello, my dog is a little""",
										    """Today, I""",
										]

										SCREAMING_SNAKE_CASE           =					tokenizer(a       ,     return_tensors="""tf"""       ,     padding=a			)
										SCREAMING_SNAKE_CASE           =					inputs["""input_ids"""]

										SCREAMING_SNAKE_CASE           =					model.generate(input_ids=a       ,     attention_mask=inputs["""attention_mask"""]			)

										SCREAMING_SNAKE_CASE           =					tokenizer(sentences[0]       ,     return_tensors="""tf"""			).input_ids
										SCREAMING_SNAKE_CASE           =					model.generate(input_ids=a			)

										SCREAMING_SNAKE_CASE           =					inputs_non_padded.shape[-1] - tf.math.reduce_sum(
										    tf.cast(inputs["""attention_mask"""][-1]       ,     tf.intaa			)			)
										SCREAMING_SNAKE_CASE           =					tokenizer(sentences[1]       ,     return_tensors="""tf"""			).input_ids
										SCREAMING_SNAKE_CASE           =					model.generate(input_ids=a       ,     max_length=model.config.max_length - num_paddings			)

										SCREAMING_SNAKE_CASE           =					tokenizer.batch_decode(a       ,     skip_special_tokens=a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(output_non_padded[0]       ,     skip_special_tokens=a			)
										SCREAMING_SNAKE_CASE           =					tokenizer.decode(output_padded[0]       ,     skip_special_tokens=a			)

										SCREAMING_SNAKE_CASE           =					[
										    """Hello, my dog is a little bit of a dork.\nI'm a little bit""",
										    """Today, I was in the middle of a conversation with a friend about the""",
										]
										self.assertListEqual(a       ,     a			)
										self.assertListEqual(a       ,     [non_padded_sentence, padded_sentence]			)







							def    _UpperCAmelCase     (					self       :	Tuple			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					"""facebook/opt-350m"""

										SCREAMING_SNAKE_CASE           =					[
										    """Today is a beautiful day and I want to""",
										    """In the city of San Francisco, the city""",
										    """Paris is the capital of France and the capital""",
										    """Computers and mobile phones have taken over the""",
										]

										SCREAMING_SNAKE_CASE           =					[]
										SCREAMING_SNAKE_CASE           =					GPTaTokenizer.from_pretrained(a			)
										SCREAMING_SNAKE_CASE           =					TFOPTForCausalLM.from_pretrained(a			)

										for prompt in self.prompts:
													SCREAMING_SNAKE_CASE           =					tokenizer(a       ,     return_tensors="""tf"""			).input_ids

													SCREAMING_SNAKE_CASE           =					model.generate(a       ,     max_length=10			)

													SCREAMING_SNAKE_CASE           =					tokenizer.batch_decode(a       ,     skip_special_tokens=a			)
													predicted_outputs += generated_string

										self.assertListEqual(a       ,     a			)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A						:							Dict							=  {
    """configuration_nllb_moe""": [
        """NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """NllbMoeConfig""",
    ]
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A						:							List[Any]							=  [
		    """NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """NllbMoeForConditionalGeneration""",
		    """NllbMoeModel""",
		    """NllbMoePreTrainedModel""",
		    """NllbMoeTop2Router""",
		    """NllbMoeSparseMLP""",
		]


if TYPE_CHECKING:
		from .configuration_nllb_moe import (
		    NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    NllbMoeConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_nllb_moe import (
				    NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    NllbMoeForConditionalGeneration,
				    NllbMoeModel,
				    NllbMoePreTrainedModel,
				    NllbMoeSparseMLP,
				    NllbMoeTopaRouter,
				)


else:
		import sys

		__A						:							Optional[Any]							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






