



"""simple docstring"""

from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel




@require_tf
class SCREAMING_SNAKE_CASE__					:
						_lowerCAmelCase							=					BlenderbotSmallConfig
						_lowerCAmelCase							=					{}
						_lowerCAmelCase							=					"""gelu"""



						def __init__(self						,     _lowercase						,     _lowercase=13						,     _lowercase=7						,     _lowercase=True						,     _lowercase=False						,     _lowercase=99						,     _lowercase=32						,     _lowercase=2						,     _lowercase=4						,     _lowercase=37						,     _lowercase=0.1						,     _lowercase=0.1						,     _lowercase=20						,     _lowercase=2						,     _lowercase=1						,     _lowercase=0						,     ):





													'''simple docstring'''

													__a		:		Any									=       parent
													__a		:		Optional[Any]									=       batch_size
													__a		:		Optional[int]									=       seq_length
													__a		:		Optional[int]									=       is_training
													__a		:		Tuple									=       use_labels
													__a		:		Dict									=       vocab_size
													__a		:		Any									=       hidden_size
													__a		:		int									=       num_hidden_layers
													__a		:		Optional[int]									=       num_attention_heads
													__a		:		Union[str, Any]									=       intermediate_size

													__a		:		List[Any]									=       hidden_dropout_prob
													__a		:		Union[str, Any]									=       attention_probs_dropout_prob
													__a		:		Dict									=       max_position_embeddings
													__a		:		List[Any]									=       eos_token_id
													__a		:		List[Any]									=       pad_token_id
													__a		:		str									=       bos_token_id



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		List[str]									=       ids_tensor([self.batch_size, self.seq_length - 1]						,     self.vocab_size							)
													__a		:		int									=       tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)						,     1							)
													__a		:		Union[str, Any]									=       tf.concat([input_ids, eos_tensor]						,     axis=1							)

													__a		:		Any									=       ids_tensor([self.batch_size, self.seq_length]						,     self.vocab_size							)

													__a		:		Dict									=       self.config_cls(
													    vocab_size=self.vocab_size						,     d_model=self.hidden_size						,     encoder_layers=self.num_hidden_layers						,     decoder_layers=self.num_hidden_layers						,     encoder_attention_heads=self.num_attention_heads						,     decoder_attention_heads=self.num_attention_heads						,     encoder_ffn_dim=self.intermediate_size						,     decoder_ffn_dim=self.intermediate_size						,     dropout=self.hidden_dropout_prob						,     attention_dropout=self.attention_probs_dropout_prob						,     max_position_embeddings=self.max_position_embeddings						,     eos_token_ids=[2]						,     bos_token_id=self.bos_token_id						,     pad_token_id=self.pad_token_id						,     decoder_start_token_id=self.pad_token_id						,     **self.config_updates						,     )
													__a		:		int									=       prepare_blenderbot_small_inputs_dict(lowercase__						,     lowercase__						,     lowercase__							)
													return config, inputs_dict





						def     lowerCAmelCase__(self						,     _lowercase						,     _lowercase							):





													'''simple docstring'''

													__a		:		Tuple									=       TFBlenderbotSmallModel(config=lowercase__							).get_decoder()
													__a		:		List[str]									=       inputs_dict["""input_ids"""]

													__a		:		int									=       input_ids[:1, :]
													__a		:		List[Any]									=       inputs_dict["""attention_mask"""][:1, :]
													__a		:		Optional[Any]									=       inputs_dict["""head_mask"""]
													__a		:		List[str]									=       1

													# first forward pass
													__a		:		Dict									=       model(lowercase__						,     attention_mask=lowercase__						,     head_mask=lowercase__						,     use_cache=lowercase__							)

													__a		:		str									=       outputs.to_tuple()

													# create hypothetical next token and extent to next_input_ids
													__a		:		str									=       ids_tensor((self.batch_size, 3)						,     config.vocab_size							)
													__a		:		str									=       tf.cast(ids_tensor((self.batch_size, 3)						,     2							)						,     tf.inta							)

													# append to next input_ids and
													__a		:		Dict									=       tf.concat([input_ids, next_tokens]						,     axis=-1							)
													__a		:		Optional[Any]									=       tf.concat([attention_mask, next_attn_mask]						,     axis=-1							)

													__a		:		Optional[int]									=       model(lowercase__						,     attention_mask=lowercase__							)[0]
													__a		:		List[str]									=       model(lowercase__						,     attention_mask=lowercase__						,     past_key_values=lowercase__							)[0]

													self.parent.assertEqual(next_tokens.shape[1]						,     output_from_past.shape[1]							)

													# select random slice
													__a		:		Optional[Any]									=       int(ids_tensor((1,)						,     output_from_past.shape[-1]							)							)
													__a		:		Optional[Any]									=       output_from_no_past[:, -3:, random_slice_idx]
													__a		:		Optional[Any]									=       output_from_past[:, :, random_slice_idx]

													# test that outputs are equal for slice
													tf.debugging.assert_near(lowercase__						,     lowercase__						,     rtol=1e-3							)







def       __magic_name__    (   _lowerCamelCase				: Any  ,       _lowerCamelCase				: int  ,       _lowerCamelCase				: str  ,       _lowerCamelCase				: Tuple=None  ,       _lowerCamelCase				: Optional[int]=None  ,       _lowerCamelCase				: Union[str, Any]=None  ,       _lowerCamelCase				: Union[str, Any]=None  ,       _lowerCamelCase				: int=None  ,       ):





							if attention_mask is None:
														__a		:		Optional[int]									=       tf.cast(tf.math.not_equal(SCREAMING_SNAKE_CASE__  ,       config.pad_token_id					)  ,       tf.inta					)
							if decoder_attention_mask is None:
														__a		:		List[Any]									=       tf.concat(
														    [
														        tf.ones(decoder_input_ids[:, :1].shape  ,       dtype=tf.inta					),
														        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]  ,       config.pad_token_id					)  ,       tf.inta					),
														    ]  ,       axis=-1  ,       )
							if head_mask is None:
														__a		:		Dict									=       tf.ones((config.encoder_layers, config.encoder_attention_heads)					)
							if decoder_head_mask is None:
														__a		:		Any									=       tf.ones((config.decoder_layers, config.decoder_attention_heads)					)
							if cross_attn_head_mask is None:
														__a		:		Optional[int]									=       tf.ones((config.decoder_layers, config.decoder_attention_heads)					)
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}




@require_tf
class SCREAMING_SNAKE_CASE__					(		_UpperCAmelCase     ,					_UpperCAmelCase     ,					unittest.TestCase      ):
						_lowerCAmelCase							=					(
						    (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
						)
						_lowerCAmelCase							=					(TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
						_lowerCAmelCase							=					(
						    {
						        """conversational""": TFBlenderbotSmallForConditionalGeneration,
						        """feature-extraction""": TFBlenderbotSmallModel,
						        """summarization""": TFBlenderbotSmallForConditionalGeneration,
						        """text2text-generation""": TFBlenderbotSmallForConditionalGeneration,
						        """translation""": TFBlenderbotSmallForConditionalGeneration,
						    }
						    if is_tf_available()
						    else {}
						)
						_lowerCAmelCase							=					True
						_lowerCAmelCase							=					False
						_lowerCAmelCase							=					False



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Any									=       TFBlenderbotSmallModelTester(self							)
													__a		:		Tuple									=       ConfigTester(self						,     config_class=lowercase__							)



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													self.config_tester.run_common_tests()





						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		str									=       self.model_tester.prepare_config_and_inputs_for_common()
													self.model_tester.check_decoder_model_past_large_inputs(*lowercase__							)







@require_tokenizers
@require_tf
class SCREAMING_SNAKE_CASE__					(		unittest.TestCase      ):
						_lowerCAmelCase							=					[
						    """Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like """
						    """  i'm going to throw up.\nand why is that?"""
						]
						_lowerCAmelCase							=					"""facebook/blenderbot_small-90M"""



						@cached_property
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													return BlenderbotSmallTokenizer.from_pretrained("""facebook/blenderbot-90M"""							)



						@cached_property
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		str									=       TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name							)
													return model





						@slow
						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		List[str]									=       self.tokenizer(self.src_text						,     return_tensors="""tf"""							)
													__a		:		List[str]									=       self.model.generate(
													    model_inputs.input_ids						,     attention_mask=model_inputs.attention_mask						,     num_beams=2						,     use_cache=lowercase__						,     )
													__a		:		Union[str, Any]									=       self.tokenizer.batch_decode(generated_ids.numpy()						,     skip_special_tokens=lowercase__							)[0]
													assert generated_words in (
													    "i don't know. i just feel like i'm going to throw up. it's not fun.",
													    "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
													    "i'm not sure. i just feel like i've been in a bad situation.",
													)






"""simple docstring"""

from unittest import TestCase

from datasets import Dataset
from minhash_deduplication import deduplicate_dataset, make_duplicate_clusters





def       __magic_name__    (   ):
							__a		:		Dict									=       {
							    """repo_name""": ["""test_repo1""", """test_repo2""", """test_repo3"""],
							    """path""": ["""test_1.py""", """test_2.py""", """unit_test.py"""],
							    """content""": ["""a """ * 2_0, """a """ * 3_0, """b """ * 7],
							}
							__a		:		Optional[Any]									=       Dataset.from_dict(_lowerCamelCase					)
							return dataset







class SCREAMING_SNAKE_CASE__					(		__snake_case      ):



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Optional[int]									=       get_dataset()
													__a		:		List[Any]									=       make_duplicate_clusters(_lowercase						,     0.85							)
													self.assertEqual(len(duplicate_clusters[0]							)						,     2							)





						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Tuple									=       get_dataset()
													__a							,     __a		:		Optional[Any]									=       deduplicate_dataset(_lowercase							)
													self.assertEqual(len(_lowercase							)						,     2							)
													print(_lowercase							)
													self.assertEqual(duplicate_clusters[0][0]["""copies"""]						,     2							)
													self.assertEqual(duplicate_clusters[0][0]["""is_extreme"""]						,     _lowercase							)

