



# flake8: noqa
# Lint as: python3

from typing import Dict, List, Optional, Type

from .. import config
from ..utils import logging
from .formatting import (
    ArrowFormatter,
    CustomFormatter,
    Formatter,
    PandasFormatter,
    PythonFormatter,
    TensorFormatter,
    format_table,
    query_table,
)
from .np_formatter import NumpyFormatter


UpperCAmelCase   :							List[str]							 =logging.get_logger(__name__)

UpperCAmelCase   :							Dict[Optional[str], Type[Formatter]]							 ={}
UpperCAmelCase   :							Dict[Optional[str], str]							 ={}
UpperCAmelCase   :							Dict[Optional[str], Exception]							 ={}



def    _lowerCAmelCase    (_lowerCAmelCase   ,	_lowerCAmelCase   ,	_lowerCAmelCase = None   ,	):
					UpperCamelCase_		  =   aliases if aliases is not None else []
					if format_type in _FORMAT_TYPES:
										logger.warning(
										    f"""Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})""")
					UpperCamelCase_		  =   formatter_cls
					for alias in set(aliases + [format_type]):
										if alias in _FORMAT_TYPES_ALIASES:
															logger.warning(
															    f"""Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})""")
										UpperCamelCase_		  =   format_type



def    _lowerCAmelCase    (_lowerCAmelCase   ,	_lowerCAmelCase   ,	_lowerCAmelCase = None):
					UpperCamelCase_		  =   aliases if aliases is not None else []
					for alias in set(aliases + [format_type]):
										UpperCamelCase_		  =   unavailable_error


# Here we define all the available formatting functions that can be used by `Dataset.set_format`
_register_formatter(PythonFormatter, None, aliases=["""python"""])
_register_formatter(ArrowFormatter, """arrow""", aliases=["""pa""", """pyarrow"""])
_register_formatter(NumpyFormatter, """numpy""", aliases=["""np"""])
_register_formatter(PandasFormatter, """pandas""", aliases=["""pd"""])
_register_formatter(CustomFormatter, """custom""")

if config.TORCH_AVAILABLE:
	from .torch_formatter import TorchFormatter

	_register_formatter(TorchFormatter, """torch""", aliases=["""pt""", """pytorch"""])
else:
	UpperCAmelCase   :							List[Any]							 =ValueError("""PyTorch needs to be installed to be able to return PyTorch tensors.""")
	_register_unavailable_formatter(_torch_error, """torch""", aliases=["""pt""", """pytorch"""])

if config.TF_AVAILABLE:
	from .tf_formatter import TFFormatter

	_register_formatter(TFFormatter, """tensorflow""", aliases=["""tf"""])
else:
	UpperCAmelCase   :							Tuple							 =ValueError("""Tensorflow needs to be installed to be able to return Tensorflow tensors.""")
	_register_unavailable_formatter(_tf_error, """tensorflow""", aliases=["""tf"""])

if config.JAX_AVAILABLE:
	from .jax_formatter import JaxFormatter

	_register_formatter(JaxFormatter, """jax""", aliases=[])
else:
	UpperCAmelCase   :							Tuple							 =ValueError("""JAX needs to be installed to be able to return JAX arrays.""")
	_register_unavailable_formatter(_jax_error, """jax""", aliases=[])



def    _lowerCAmelCase    (_lowerCAmelCase):
					if format_type in _FORMAT_TYPES_ALIASES:
										return _FORMAT_TYPES_ALIASES[format_type]
					else:
										return format_type






def    _lowerCAmelCase    (_lowerCAmelCase   ,	**_lowerCAmelCase):
					UpperCamelCase_		  =   get_format_type_from_alias(_lowerCAmelCase)
					if format_type in _FORMAT_TYPES:
										return _FORMAT_TYPES[format_type](**_lowerCAmelCase)
					if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
										raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
					else:
										raise ValueError(
										    f"""Return type should be None or selected in {list(type for type in _FORMAT_TYPES.keys() if type != None)}, but got '{format_type}'""")





from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel



def    _lowerCAmelCase    (_lowerCAmelCase   ,	_lowerCAmelCase   ,	_lowerCAmelCase=None   ,	_lowerCAmelCase=None):
					if attention_mask is None:
										UpperCamelCase_		  =   tf.cast(tf.math.not_equal(_lowerCAmelCase   ,	config.pad_token_id)   ,	tf.inta)
					return {"input_ids": input_ids, "attention_mask": attention_mask}


@require_tf
class       _lowercase   :
				'''simple docstring'''







				lowercase__   						=						OPTConfig
				lowercase__   						=						{}
				lowercase__   						=						"""gelu"""

				def __init__( self     , snake_case__     , snake_case__=13     , snake_case__=7     , snake_case__=True     , snake_case__=False     , snake_case__=99     , snake_case__=16     , snake_case__=2     , snake_case__=4     , snake_case__=4     , snake_case__="gelu"     , snake_case__=0.1     , snake_case__=0.1     , snake_case__=20     , snake_case__=2     , snake_case__=1     , snake_case__=0     , snake_case__=16     , snake_case__=16     , ):

									'''simple docstring'''


									UpperCamelCase_		  =   parent
									UpperCamelCase_		  =   batch_size
									UpperCamelCase_		  =   seq_length
									UpperCamelCase_		  =   is_training
									UpperCamelCase_		  =   use_labels
									UpperCamelCase_		  =   vocab_size
									UpperCamelCase_		  =   hidden_size
									UpperCamelCase_		  =   num_hidden_layers
									UpperCamelCase_		  =   num_attention_heads
									UpperCamelCase_		  =   intermediate_size
									UpperCamelCase_		  =   hidden_act
									UpperCamelCase_		  =   hidden_dropout_prob
									UpperCamelCase_		  =   attention_probs_dropout_prob
									UpperCamelCase_		  =   max_position_embeddings
									UpperCamelCase_		  =   eos_token_id
									UpperCamelCase_		  =   pad_token_id
									UpperCamelCase_		  =   bos_token_id
									UpperCamelCase_		  =   embed_dim
									UpperCamelCase_		  =   word_embed_proj_dim
									UpperCamelCase_		  =   False

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   ids_tensor([self.batch_size, self.seq_length - 1]     , self.vocab_size    )
									UpperCamelCase_		  =   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size    )     , 1    )
									UpperCamelCase_		  =   tf.concat([input_ids, eos_tensor]     , axis=1    )

									UpperCamelCase_		  =   self.config_cls(
									    vocab_size=self.vocab_size     , hidden_size=self.hidden_size     , num_hidden_layers=self.num_hidden_layers     , num_attention_heads=self.num_attention_heads     , ffn_dim=self.intermediate_size     , dropout=self.hidden_dropout_prob     , attention_dropout=self.attention_probs_dropout_prob     , max_position_embeddings=self.max_position_embeddings     , eos_token_id=self.eos_token_id     , bos_token_id=self.bos_token_id     , pad_token_id=self.pad_token_id     , embed_dim=self.embed_dim     , word_embed_proj_dim=self.word_embed_proj_dim     , is_encoder_decoder=snake_case__     , **self.config_updates     , )
									UpperCamelCase_		  =   prepare_opt_inputs_dict(snake_case__     , snake_case__    )
									return config, inputs_dict

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTModel(config=snake_case__    )
									UpperCamelCase_		  =   inputs_dict["input_ids"]

									UpperCamelCase_		  =   input_ids[:1, :]
									UpperCamelCase_		  =   inputs_dict["attention_mask"][:1, :]
									UpperCamelCase_		  =   1

									# first forward pass
									UpperCamelCase_		  =   model(snake_case__     , attention_mask=snake_case__     , use_cache=snake_case__    )

									UpperCamelCase_  ,    UpperCamelCase_		  =   outputs.to_tuple()

									# create hypothetical next token and extent to next_input_ids
									UpperCamelCase_		  =   ids_tensor((self.batch_size, 3)     , config.vocab_size    )
									UpperCamelCase_		  =   tf.cast(ids_tensor((self.batch_size, 3)     , 2    )     , tf.inta    )

									# append to next input_ids and
									UpperCamelCase_		  =   tf.concat([input_ids, next_tokens]     , axis=-1    )
									UpperCamelCase_		  =   tf.concat([attention_mask, next_attn_mask]     , axis=-1    )

									UpperCamelCase_		  =   model(snake_case__     , attention_mask=snake_case__    )[0]
									UpperCamelCase_		  =   model(snake_case__     , attention_mask=snake_case__     , past_key_values=snake_case__    )[0]

									self.parent.assertEqual(next_tokens.shape[1]     , output_from_past.shape[1]    )

									# select random slice
									UpperCamelCase_		  =   int(ids_tensor((1,)     , output_from_past.shape[-1]    )    )
									UpperCamelCase_		  =   output_from_no_past[:, -3:, random_slice_idx]
									UpperCamelCase_		  =   output_from_past[:, :, random_slice_idx]

									# test that outputs are equal for slice
									tf.debugging.assert_near(snake_case__     , snake_case__     , rtol=1e-3    )




@require_tf
class       _lowercase   (a_      ,   a_      ,   unittest.TestCase      ):
				'''simple docstring'''







				lowercase__   						=						(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
				lowercase__   						=						(TFOPTForCausalLM,) if is_tf_available() else ()
				lowercase__   						=						(
				    {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
				)
				lowercase__   						=						False
				lowercase__   						=						False
				lowercase__   						=						False
				lowercase__   						=						10

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTModelTester(self    )
									UpperCamelCase_		  =   ConfigTester(self     , config_class=snake_case__    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									self.config_tester.run_common_tests()

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_decoder_model_past_large_inputs(*snake_case__    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_  ,    UpperCamelCase_		  =   self.model_tester.prepare_config_and_inputs_for_common()

									def _get_word_embedding_weight(snake_case__     , snake_case__    ):
														if hasattr(snake_case__     , "weight"    ):
																			return embedding_layer.weight
														else:
																			# Here we build the word embeddings weights if not exists.
																			# And then we retry to get the attribute once built.
																			model.build()
																			if hasattr(snake_case__     , "weight"    ):
																								return embedding_layer.weight
																			else:
																								return None

									for model_class in self.all_model_classes:
														for size in [config.vocab_size - 10, config.vocab_size + 10]:
																			# build the embeddings
																			UpperCamelCase_		  =   model_class(config=snake_case__    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_input_embeddings()    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_output_embeddings()    )

																			# reshape the embeddings
																			model.resize_token_embeddings(snake_case__    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_input_embeddings()    )
																			UpperCamelCase_		  =   _get_word_embedding_weight(snake_case__     , model.get_output_embeddings()    )

																			# check that the resized embeddings size matches the desired size.
																			UpperCamelCase_		  =   size if size is not None else config.vocab_size

																			self.assertEqual(new_input_embeddings.shape[0]     , snake_case__    )

																			# check that weights remain the same after resizing
																			UpperCamelCase_		  =   True
																			for pa, pa in zip(old_input_embeddings.value()     , new_input_embeddings.value()    ):
																								if tf.math.reduce_sum(tf.math.abs(pa - pa    )    ) > 0:
																													UpperCamelCase_		  =   False
																			self.assertTrue(snake_case__    )

																			if old_output_embeddings is not None and new_output_embeddings is not None:
																								self.assertEqual(new_output_embeddings.shape[0]     , snake_case__    )

																								UpperCamelCase_		  =   True
																								for pa, pa in zip(old_output_embeddings.value()     , new_output_embeddings.value()    ):
																													if tf.math.reduce_sum(tf.math.abs(pa - pa    )    ) > 0:
																																		UpperCamelCase_		  =   False
																								self.assertTrue(snake_case__    )




def    _lowerCAmelCase    (_lowerCAmelCase):
					return tf.constant(_lowerCAmelCase   ,	dtype=tf.intaa)


@require_tf
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				lowercase__   						=						99

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   tf.ones((4, 1)     , dtype=tf.intaa    ) * 2
									UpperCamelCase_		  =   tf.concat([ids_tensor((4, 6)     , self.vocab_size - 3    ) + 3, eos_column_vector]     , axis=1    )
									UpperCamelCase_		  =   input_ids.shape[0]
									UpperCamelCase_		  =   OPTConfig(
									    vocab_size=self.vocab_size     , hidden_size=24     , num_hidden_layers=2     , num_attention_heads=2     , ffn_dim=32     , max_position_embeddings=48     , eos_token_id=2     , pad_token_id=1     , bos_token_id=0     , )
									return config, input_ids, batch_size




@require_sentencepiece
@require_tf
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				@slow
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTModel.from_pretrained("facebook/opt-350m"    )
									UpperCamelCase_		  =   _long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]    )
									UpperCamelCase_		  =   tf.not_equal(snake_case__     , model.config.pad_token_id    )
									with tf.GradientTape():
														UpperCamelCase_		  =   model(input_ids=snake_case__     , attention_mask=snake_case__    ).last_hidden_state
									UpperCamelCase_		  =   (1, 11, 512)
									self.assertEqual(output.shape     , snake_case__    )
									UpperCamelCase_		  =   tf.constant(
									    [[-0.2_873, -1.9_218, -0.3_033], [-1.2_710, -0.1_338, -0.1_902], [0.4_095, 0.1_214, -1.3_121]]    )
									self.assertTrue(np.allclose(output[:, :3, :3]     , snake_case__     , atol=4e-3    )    )

									UpperCamelCase_		  =   tf.function(snake_case__     , jit_compile=snake_case__    )
									UpperCamelCase_		  =   xla_generate(snake_case__     , snake_case__    )[0]
									self.assertTrue(np.allclose(output[:, :3, :3]     , snake_case__     , atol=4e-2    )    )




@require_tf
@slow
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									super().setUp()
									UpperCamelCase_		  =   "facebook/opt-350m"

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(self.path_model    )
									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(self.path_model    )

									UpperCamelCase_		  =   [
									    "Today is a beautiful day and I want to",
									    "In the city of",
									    "Paris is the capital of France and",
									    "Computers and mobile phones have taken",
									]
									# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
									UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"     , padding=snake_case__     , add_special_tokens=snake_case__    )
									UpperCamelCase_		  =   tf.math.reduce_mean(model(inputs.input_ids     , attention_mask=inputs.attention_mask    )[0]     , axis=-1    )
									UpperCamelCase_		  =   tf.constant(
									    [
									        [1.3_851, -13.8_923, -10.5_229, -10.7_533, -0.2_309, -10.2_384, -0.5_365, -9.0_947, -5.1_670],
									        [-4.7_073, -10.6_276, -3.9_415, -21.5_242, -0.2_822, -0.2_822, -0.2_822, -0.2_822, -0.2_822],
									        [0.6_247, -3.4_229, -8.9_179, -1.4_297, -14.1_650, 1.4_146, -9.0_218, -0.2_703, -0.2_703],
									        [6.4_783, -1.9_913, -10.7_926, -2.3_336, 1.5_092, -0.9_974, -6.8_213, 1.3_477, 1.3_477],
									    ]    )
									self.assertTrue(np.allclose(snake_case__     , snake_case__     , atol=1e-4    )    )

									UpperCamelCase_		  =   tf.function(snake_case__     , jit_compile=snake_case__    )
									UpperCamelCase_		  =   tf.math.reduce_mean(xla_generate(inputs.input_ids     , attention_mask=inputs.attention_mask    )[0]     , axis=-1    )
									self.assertTrue(np.allclose(snake_case__     , snake_case__     , atol=1e-4    )    )




@require_tf
@slow
class       _lowercase   (unittest.TestCase      ):
				'''simple docstring'''







				@property
				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									return [
									    "Today is a beautiful day and I want",
									    "In the city of",
									    "Paris is the capital of France and",
									    "Computers and mobile phones have taken",
									]

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   "facebook/opt-125m"

									UpperCamelCase_		  =   [
									    "Today is a beautiful day and I want to",
									    "In the city of New York, the city",
									    "Paris is the capital of France and the capital",
									    "Computers and mobile phones have taken over the",
									]

									UpperCamelCase_		  =   []
									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(snake_case__    )
									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(snake_case__    )

									for prompt in self.prompts:
														UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"    ).input_ids

														UpperCamelCase_		  =   model.generate(snake_case__     , max_length=10    )

														UpperCamelCase_		  =   tokenizer.batch_decode(snake_case__     , skip_special_tokens=snake_case__    )
														predicted_outputs += generated_string

									self.assertListEqual(snake_case__     , snake_case__    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   "facebook/opt-350m"

									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(snake_case__    )
									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(snake_case__    )

									UpperCamelCase_		  =   "left"

									# use different length sentences to test batching
									UpperCamelCase_		  =   [
									    "Hello, my dog is a little",
									    "Today, I",
									]

									UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"     , padding=snake_case__    )
									UpperCamelCase_		  =   inputs["input_ids"]

									UpperCamelCase_		  =   model.generate(input_ids=snake_case__     , attention_mask=inputs["attention_mask"]    )

									UpperCamelCase_		  =   tokenizer(sentences[0]     , return_tensors="tf"    ).input_ids
									UpperCamelCase_		  =   model.generate(input_ids=snake_case__    )

									UpperCamelCase_		  =   inputs_non_padded.shape[-1] - tf.math.reduce_sum(
									    tf.cast(inputs["attention_mask"][-1]     , tf.intaa    )    )
									UpperCamelCase_		  =   tokenizer(sentences[1]     , return_tensors="tf"    ).input_ids
									UpperCamelCase_		  =   model.generate(input_ids=snake_case__     , max_length=model.config.max_length - num_paddings    )

									UpperCamelCase_		  =   tokenizer.batch_decode(snake_case__     , skip_special_tokens=snake_case__    )
									UpperCamelCase_		  =   tokenizer.decode(output_non_padded[0]     , skip_special_tokens=snake_case__    )
									UpperCamelCase_		  =   tokenizer.decode(output_padded[0]     , skip_special_tokens=snake_case__    )

									UpperCamelCase_		  =   [
									    "Hello, my dog is a little bit of a dork.\nI'm a little bit",
									    "Today, I was in the middle of a conversation with a friend about the",
									]
									self.assertListEqual(snake_case__     , snake_case__    )
									self.assertListEqual(snake_case__     , [non_padded_sentence, padded_sentence]    )

				def       _lowerCamelCase					( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   "facebook/opt-350m"

									UpperCamelCase_		  =   [
									    "Today is a beautiful day and I want to",
									    "In the city of San Francisco, the city",
									    "Paris is the capital of France and the capital",
									    "Computers and mobile phones have taken over the",
									]

									UpperCamelCase_		  =   []
									UpperCamelCase_		  =   GPTaTokenizer.from_pretrained(snake_case__    )
									UpperCamelCase_		  =   TFOPTForCausalLM.from_pretrained(snake_case__    )

									for prompt in self.prompts:
														UpperCamelCase_		  =   tokenizer(snake_case__     , return_tensors="tf"    ).input_ids

														UpperCamelCase_		  =   model.generate(snake_case__     , max_length=10    )

														UpperCamelCase_		  =   tokenizer.batch_decode(snake_case__     , skip_special_tokens=snake_case__    )
														predicted_outputs += generated_string

									self.assertListEqual(snake_case__     , snake_case__    )
