from __future__ import annotations

import unittest

from transformers import LEDConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import TFLEDForConditionalGeneration, TFLEDModel


@require_tf
class 			lowerCamelCase_  :






    '''simple docstring'''






    a__			:		List[Any]      			=   LEDConfig
    a__			:		int      			=   {}
    a__			:		Any      			=   """gelu"""







    def __init__(			self				,				__lowercase				,				__lowercase=13				,				__lowercase=7				,				__lowercase=True				,				__lowercase=False				,				__lowercase=99				,				__lowercase=32				,				__lowercase=2				,				__lowercase=4				,				__lowercase=37				,				__lowercase=0.1				,				__lowercase=0.1				,				__lowercase=20				,				__lowercase=2				,				__lowercase=1				,				__lowercase=0				,				__lowercase=4				,				) ->			str:
        __UpperCamelCase  :Optional[int]    						=	parent
        __UpperCamelCase  :str    						=	batch_size
        __UpperCamelCase  :Union[str, Any]    						=	seq_length
        __UpperCamelCase  :List[str]    						=	is_training
        __UpperCamelCase  :Union[str, Any]    						=	use_labels
        __UpperCamelCase  :Optional[int]    						=	vocab_size
        __UpperCamelCase  :List[Any]    						=	hidden_size
        __UpperCamelCase  :Dict    						=	num_hidden_layers
        __UpperCamelCase  :Union[str, Any]    						=	num_attention_heads
        __UpperCamelCase  :Union[str, Any]    						=	intermediate_size

        __UpperCamelCase  :Any    						=	hidden_dropout_prob
        __UpperCamelCase  :str    						=	attention_probs_dropout_prob
        __UpperCamelCase  :int    						=	max_position_embeddings
        __UpperCamelCase  :Union[str, Any]    						=	eos_token_id
        __UpperCamelCase  :Dict    						=	pad_token_id
        __UpperCamelCase  :List[Any]    						=	bos_token_id
        __UpperCamelCase  :List[Any]    						=	attention_window

        # `ModelTesterMixin.test_attention_outputs` is expecting attention tensors to be of size
        # [num_attention_heads, encoder_seq_length, encoder_key_length], but TFLongformerSelfAttention
        # returns attention of shape [num_attention_heads, encoder_seq_length, self.attention_window + 1]
        # because its local attention only attends to `self.attention_window` and one before and one after
        __UpperCamelCase  :Tuple    						=	self.attention_window + 2

        # because of padding `encoder_seq_length`, is different from `seq_length`. Relevant for
        # the `test_attention_outputs` and `test_hidden_states_output` tests
        __UpperCamelCase  :Union[str, Any]    						=	(
            self.seq_length + (self.attention_window - self.seq_length % self.attention_window) % self.attention_window
        )







    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :Any    						=	ids_tensor([self.batch_size, self.seq_length - 1]				,				self.vocab_size)
        __UpperCamelCase  :Tuple    						=	tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size)				,				1)
        __UpperCamelCase  :Dict    						=	tf.concat([input_ids, eos_tensor]				,				axis=1)

        __UpperCamelCase  :List[str]    						=	ids_tensor([self.batch_size, self.seq_length]				,				self.vocab_size)

        __UpperCamelCase  :Union[str, Any]    						=	self.config_cls(
            vocab_size=self.vocab_size				,				d_model=self.hidden_size				,				encoder_layers=self.num_hidden_layers				,				decoder_layers=self.num_hidden_layers				,				encoder_attention_heads=self.num_attention_heads				,				decoder_attention_heads=self.num_attention_heads				,				encoder_ffn_dim=self.intermediate_size				,				decoder_ffn_dim=self.intermediate_size				,				dropout=self.hidden_dropout_prob				,				attention_dropout=self.attention_probs_dropout_prob				,				max_position_embeddings=self.max_position_embeddings				,				eos_token_ids=[2]				,				bos_token_id=self.bos_token_id				,				pad_token_id=self.pad_token_id				,				decoder_start_token_id=self.pad_token_id				,				attention_window=self.attention_window				,				**self.config_updates				,				)
        __UpperCamelCase  :Tuple    						=	prepare_led_inputs_dict(__UpperCamelCase				,				__UpperCamelCase				,				__UpperCamelCase)
        __UpperCamelCase  :Tuple    						=	tf.concat(
            [tf.zeros_like(__UpperCamelCase)[:, :-1], tf.ones_like(__UpperCamelCase)[:, -1:]]				,				axis=-1				,				)
        __UpperCamelCase  :Union[str, Any]    						=	global_attention_mask
        return config, inputs_dict







    def     UpperCamelCase__     (			self				,				__lowercase				,				__lowercase) ->			Optional[Any]:
        __UpperCamelCase  :List[str]    						=	TFLEDModel(config=__UpperCamelCase).get_decoder()
        __UpperCamelCase  :Dict    						=	inputs_dict['''input_ids''']

        __UpperCamelCase  :str    						=	input_ids[:1, :]
        __UpperCamelCase  :List[Any]    						=	inputs_dict['''attention_mask'''][:1, :]
        __UpperCamelCase  :int    						=	1

        # first forward pass
        __UpperCamelCase  :Optional[Any]    						=	model(__UpperCamelCase				,				attention_mask=__UpperCamelCase				,				use_cache=__UpperCamelCase)

        __UpperCamelCase						, __UpperCamelCase  :List[Any]    						=	outputs.to_tuple()

        # create hypothetical next token and extent to next_input_ids
        __UpperCamelCase  :Union[str, Any]    						=	ids_tensor((self.batch_size, 3)				,				config.vocab_size)
        __UpperCamelCase  :Tuple    						=	tf.cast(ids_tensor((self.batch_size, 3)				,				2)				,				tf.inta)

        # append to next input_ids and
        __UpperCamelCase  :List[Any]    						=	tf.concat([input_ids, next_tokens]				,				axis=-1)
        __UpperCamelCase  :int    						=	tf.concat([attention_mask, next_attn_mask]				,				axis=-1)

        __UpperCamelCase  :Tuple    						=	model(__UpperCamelCase				,				attention_mask=__UpperCamelCase)[0]
        __UpperCamelCase  :Dict    						=	model(__UpperCamelCase				,				attention_mask=__UpperCamelCase				,				past_key_values=__UpperCamelCase)[0]

        self.parent.assertEqual(next_tokens.shape[1]				,				output_from_past.shape[1])

        # select random slice
        __UpperCamelCase  :Tuple    						=	int(ids_tensor((1,)				,				output_from_past.shape[-1]))
        __UpperCamelCase  :List[str]    						=	output_from_no_past[:, -3:, random_slice_idx]
        __UpperCamelCase  :Optional[Any]    						=	output_from_past[:, :, random_slice_idx]

        # test that outputs are equal for slice
        tf.debugging.assert_near(__UpperCamelCase				,				__UpperCamelCase				,				rtol=1E-3)



def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE=None						,    SCREAMING_SNAKE_CASE=None						,    SCREAMING_SNAKE_CASE=None						,    SCREAMING_SNAKE_CASE=None						,    ):
    if attention_mask is None:
        __UpperCamelCase  :Optional[Any]    						=	tf.cast(tf.math.not_equal(lowercase__						,    config.pad_token_id			)						,    tf.inta			)
    if decoder_attention_mask is None:
        __UpperCamelCase  :Any    						=	tf.concat(
            [
                tf.ones(decoder_input_ids[:, :1].shape						,    dtype=tf.inta			),
                tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]						,    config.pad_token_id			)						,    tf.inta			),
            ]						,    axis=-1						,    )
    if head_mask is None:
        __UpperCamelCase  :Union[str, Any]    						=	tf.ones((config.encoder_layers, config.encoder_attention_heads)			)
    if decoder_head_mask is None:
        __UpperCamelCase  :Dict    						=	tf.ones((config.decoder_layers, config.decoder_attention_heads)			)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "decoder_input_ids": decoder_input_ids,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
    }


@require_tf
class 			lowerCamelCase_  (     __snake_case		,			__snake_case		,			unittest.TestCase				):






    '''simple docstring'''






    a__			:		Union[str, Any]      			=   (TFLEDForConditionalGeneration, TFLEDModel) if is_tf_available() else ()
    a__			:		Dict      			=   (TFLEDForConditionalGeneration,) if is_tf_available() else ()
    a__			:		List[Any]      			=   (
        {
            """conversational""": TFLEDForConditionalGeneration,
            """feature-extraction""": TFLEDModel,
            """summarization""": TFLEDForConditionalGeneration,
            """text2text-generation""": TFLEDForConditionalGeneration,
            """translation""": TFLEDForConditionalGeneration,
        }
        if is_tf_available()
        else {}
    )
    a__			:		List[str]      			=   True
    a__			:		int      			=   False
    a__			:		Optional[Any]      			=   False
    a__			:		Any      			=   False







    def     UpperCamelCase__     (			self) ->			Tuple:
        __UpperCamelCase  :List[str]    						=	TFLEDModelTester(self)
        __UpperCamelCase  :Union[str, Any]    						=	ConfigTester(self				,				config_class=__UpperCamelCase)







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        self.config_tester.run_common_tests()







    def     UpperCamelCase__     (			self) ->			Union[str, Any]:
        __UpperCamelCase  :Union[str, Any]    						=	self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.check_decoder_model_past_large_inputs(*__UpperCamelCase)







    def     UpperCamelCase__     (			self) ->			Dict:
        __UpperCamelCase						, __UpperCamelCase  :Optional[Any]    						=	self.model_tester.prepare_config_and_inputs_for_common()
        __UpperCamelCase  :Optional[Any]    						=	tf.zeros_like(inputs_dict['''attention_mask'''])
        __UpperCamelCase  :List[Any]    						=	2
        __UpperCamelCase  :int    						=	tf.where(
            tf.range(self.model_tester.seq_length)[None, :] < num_global_attn_indices				,				1				,				inputs_dict['''global_attention_mask''']				,				)

        __UpperCamelCase  :Dict    						=	True
        __UpperCamelCase  :Dict    						=	self.model_tester.seq_length
        __UpperCamelCase  :List[Any]    						=	self.model_tester.encoder_seq_length

        def check_decoder_attentions_output(__lowercase):
            __UpperCamelCase  :int    						=	outputs.decoder_attentions
            self.assertEqual(len(__UpperCamelCase)				,				self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(decoder_attentions[0].shape[-3:])				,				[self.model_tester.num_attention_heads, seq_length, seq_length]				,				)

        def check_encoder_attentions_output(__lowercase):
            __UpperCamelCase  :int    						=	[t.numpy() for t in outputs.encoder_attentions]
            __UpperCamelCase  :Any    						=	[t.numpy() for t in outputs.encoder_global_attentions]
            self.assertEqual(len(__UpperCamelCase)				,				self.model_tester.num_hidden_layers)
            self.assertEqual(len(__UpperCamelCase)				,				self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(attentions[0].shape[-3:])				,				[self.model_tester.num_attention_heads, seq_length, seq_length]				,				)
            self.assertListEqual(
                list(global_attentions[0].shape[-3:])				,				[self.model_tester.num_attention_heads, encoder_seq_length, num_global_attn_indices]				,				)

        for model_class in self.all_model_classes:
            __UpperCamelCase  :Tuple    						=	True
            __UpperCamelCase  :int    						=	False
            __UpperCamelCase  :Optional[int]    						=	False
            __UpperCamelCase  :List[Any]    						=	model_class(__UpperCamelCase)
            __UpperCamelCase  :int    						=	model(self._prepare_for_class(__UpperCamelCase				,				__UpperCamelCase))
            __UpperCamelCase  :Dict    						=	len(__UpperCamelCase)
            self.assertEqual(config.output_hidden_states				,				__UpperCamelCase)
            check_encoder_attentions_output(__UpperCamelCase)

            if self.is_encoder_decoder:
                __UpperCamelCase  :Dict    						=	model_class(__UpperCamelCase)
                __UpperCamelCase  :Optional[int]    						=	model(self._prepare_for_class(__UpperCamelCase				,				__UpperCamelCase))
                self.assertEqual(config.output_hidden_states				,				__UpperCamelCase)
                check_decoder_attentions_output(__UpperCamelCase)

            # Check that output attentions can also be changed via the config
            del inputs_dict["output_attentions"]
            __UpperCamelCase  :Optional[Any]    						=	True
            __UpperCamelCase  :str    						=	model_class(__UpperCamelCase)
            __UpperCamelCase  :Optional[int]    						=	model(self._prepare_for_class(__UpperCamelCase				,				__UpperCamelCase))
            self.assertEqual(config.output_hidden_states				,				__UpperCamelCase)
            check_encoder_attentions_output(__UpperCamelCase)

            # Check attention is always last and order is fine
            __UpperCamelCase  :Dict    						=	True
            __UpperCamelCase  :int    						=	True
            __UpperCamelCase  :int    						=	model_class(__UpperCamelCase)
            __UpperCamelCase  :Dict    						=	model(self._prepare_for_class(__UpperCamelCase				,				__UpperCamelCase))

            self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)				,				len(__UpperCamelCase))
            self.assertEqual(model.config.output_hidden_states				,				__UpperCamelCase)
            check_encoder_attentions_output(__UpperCamelCase)







    @unittest.skip('''LED keeps using potentially symbolic tensors in conditionals and breaks tracing.''')
    def     UpperCamelCase__     (			self) ->			List[Any]:
        pass







    def     UpperCamelCase__     (			self) ->			Union[str, Any]:
        pass



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    return tf.constant(lowercase__						,    dtype=tf.intaa			)


__lowercase											=      1e-4


@slow
@require_tf
class 			lowerCamelCase_  (     unittest.TestCase				):






    '''simple docstring'''






    def     UpperCamelCase__     (			self) ->			Union[str, Any]:
        __UpperCamelCase  :Optional[int]    						=	TFLEDForConditionalGeneration.from_pretrained('''allenai/led-base-16384''').led

        # change to intended input here
        __UpperCamelCase  :int    						=	_long_tensor([512 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]])
        __UpperCamelCase  :Optional[int]    						=	_long_tensor([128 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]])
        __UpperCamelCase  :List[Any]    						=	prepare_led_inputs_dict(model.config				,				__UpperCamelCase				,				__UpperCamelCase)
        __UpperCamelCase  :int    						=	model(**__UpperCamelCase)[0]
        __UpperCamelCase  :List[str]    						=	(1, 1_024, 768)
        self.assertEqual(output.shape				,				__UpperCamelCase)
        # change to expected output here
        __UpperCamelCase  :List[str]    						=	tf.convert_to_tensor(
            [[2.30_50, 2.82_79, 0.65_31], [-1.84_57, -0.14_55, -3.56_61], [-1.01_86, 0.45_86, -2.20_43]]				,				)
        tf.debugging.assert_near(output[:, :3, :3]				,				__UpperCamelCase				,				atol=1E-3)







    def     UpperCamelCase__     (			self) ->			Tuple:
        __UpperCamelCase  :str    						=	TFLEDForConditionalGeneration.from_pretrained('''allenai/led-base-16384''')

        # change to intended input here
        __UpperCamelCase  :Any    						=	_long_tensor([512 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]])
        __UpperCamelCase  :Optional[Any]    						=	_long_tensor([128 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]])
        __UpperCamelCase  :str    						=	prepare_led_inputs_dict(model.config				,				__UpperCamelCase				,				__UpperCamelCase)
        __UpperCamelCase  :Optional[int]    						=	model(**__UpperCamelCase)[0]
        __UpperCamelCase  :Optional[int]    						=	(1, 1_024, model.config.vocab_size)
        self.assertEqual(output.shape				,				__UpperCamelCase)
        # change to expected output here
        __UpperCamelCase  :Optional[Any]    						=	tf.convert_to_tensor(
            [[33.65_07, 6.45_72, 16.80_89], [5.87_39, -2.42_38, 11.29_02], [-3.21_39, -4.31_49, 4.27_83]]				,				)
        tf.debugging.assert_near(output[:, :3, :3]				,				__UpperCamelCase				,				atol=1E-3				,				rtol=1E-3)






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase											=      logging.get_logger(__name__)

__lowercase											=      {
    '''microsoft/unispeech-large-1500h-cv''': (
        '''https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'''
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}


class 			lowerCamelCase_  (     UpperCAmelCase_				):






    '''simple docstring'''






    a__			:		Dict      			=   """unispeech"""







    def __init__(			self				,				__lowercase=32				,				__lowercase=768				,				__lowercase=12				,				__lowercase=12				,				__lowercase=3_072				,				__lowercase="gelu"				,				__lowercase=0.1				,				__lowercase=0.1				,				__lowercase=0.1				,				__lowercase=0.0				,				__lowercase=0.0				,				__lowercase=0.1				,				__lowercase=0.1				,				__lowercase=0.02				,				__lowercase=1E-5				,				__lowercase="group"				,				__lowercase="gelu"				,				__lowercase=(512, 512, 512, 512, 512, 512, 512)				,				__lowercase=(5, 2, 2, 2, 2, 2, 2)				,				__lowercase=(10, 3, 3, 3, 3, 2, 2)				,				__lowercase=False				,				__lowercase=128				,				__lowercase=16				,				__lowercase=False				,				__lowercase=True				,				__lowercase=0.05				,				__lowercase=10				,				__lowercase=2				,				__lowercase=0.0				,				__lowercase=10				,				__lowercase=0				,				__lowercase=320				,				__lowercase=2				,				__lowercase=0.1				,				__lowercase=100				,				__lowercase=256				,				__lowercase=256				,				__lowercase=0.1				,				__lowercase="mean"				,				__lowercase=False				,				__lowercase=False				,				__lowercase=256				,				__lowercase=80				,				__lowercase=0				,				__lowercase=1				,				__lowercase=2				,				__lowercase=0.5				,				**__lowercase				,				) ->			Optional[int]:
        super().__init__(**__lowercase				,				pad_token_id=__lowercase				,				bos_token_id=__lowercase				,				eos_token_id=__lowercase)
        __UpperCamelCase  :str    						=	hidden_size
        __UpperCamelCase  :List[str]    						=	feat_extract_norm
        __UpperCamelCase  :str    						=	feat_extract_activation
        __UpperCamelCase  :str    						=	list(__lowercase)
        __UpperCamelCase  :List[Any]    						=	list(__lowercase)
        __UpperCamelCase  :List[Any]    						=	list(__lowercase)
        __UpperCamelCase  :Any    						=	conv_bias
        __UpperCamelCase  :List[Any]    						=	num_conv_pos_embeddings
        __UpperCamelCase  :Tuple    						=	num_conv_pos_embedding_groups
        __UpperCamelCase  :Optional[int]    						=	len(self.conv_dim)
        __UpperCamelCase  :Optional[int]    						=	num_hidden_layers
        __UpperCamelCase  :Union[str, Any]    						=	intermediate_size
        __UpperCamelCase  :Tuple    						=	hidden_act
        __UpperCamelCase  :Optional[int]    						=	num_attention_heads
        __UpperCamelCase  :Any    						=	hidden_dropout
        __UpperCamelCase  :List[str]    						=	attention_dropout
        __UpperCamelCase  :int    						=	activation_dropout
        __UpperCamelCase  :int    						=	feat_proj_dropout
        __UpperCamelCase  :Any    						=	final_dropout
        __UpperCamelCase  :Optional[Any]    						=	layerdrop
        __UpperCamelCase  :Any    						=	layer_norm_eps
        __UpperCamelCase  :List[str]    						=	initializer_range
        __UpperCamelCase  :Tuple    						=	num_ctc_classes
        __UpperCamelCase  :Union[str, Any]    						=	vocab_size
        __UpperCamelCase  :List[Any]    						=	do_stable_layer_norm
        __UpperCamelCase  :Dict    						=	use_weighted_layer_sum
        __UpperCamelCase  :str    						=	classifier_proj_size

        if (
            (len(self.conv_stride) != self.num_feat_extract_layers)
            or (len(self.conv_kernel) != self.num_feat_extract_layers)
            or (len(self.conv_dim) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                f""" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"""
                f""" `len(config.conv_kernel) = {len(self.conv_kernel)}`.""")

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        __UpperCamelCase  :List[Any]    						=	apply_spec_augment
        __UpperCamelCase  :Optional[int]    						=	mask_time_prob
        __UpperCamelCase  :int    						=	mask_time_length
        __UpperCamelCase  :Any    						=	mask_time_min_masks
        __UpperCamelCase  :Any    						=	mask_feature_prob
        __UpperCamelCase  :str    						=	mask_feature_length
        __UpperCamelCase  :Tuple    						=	mask_feature_min_masks

        # parameters for pretraining with codevector quantized representations
        __UpperCamelCase  :Optional[Any]    						=	num_codevectors_per_group
        __UpperCamelCase  :Dict    						=	num_codevector_groups
        __UpperCamelCase  :Optional[int]    						=	contrastive_logits_temperature
        __UpperCamelCase  :Union[str, Any]    						=	feat_quantizer_dropout
        __UpperCamelCase  :List[str]    						=	num_negatives
        __UpperCamelCase  :Union[str, Any]    						=	codevector_dim
        __UpperCamelCase  :int    						=	proj_codevector_dim
        __UpperCamelCase  :Tuple    						=	diversity_loss_weight

        # ctc loss
        __UpperCamelCase  :List[Any]    						=	ctc_loss_reduction
        __UpperCamelCase  :int    						=	ctc_zero_infinity

        # pretraining loss
        __UpperCamelCase  :Optional[Any]    						=	replace_prob







    @property
    def     UpperCamelCase__     (			self) ->			Dict:
        return functools.reduce(operator.mul				,				self.conv_stride				,				1)





