





import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase :  str										=  get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
 import sentencepiece as sp


__lowerCamelCase :  int										=  5
__lowerCamelCase :  Dict										=  10



@require_sentencepiece
@require_tokenizers
class       A__     ( _snake_case ,     unittest.TestCase    ):
      _UpperCAmelCase			:str							=     SpeechaTextTokenizer
      _UpperCAmelCase			:int							=     False
      _UpperCAmelCase			:List[str]							=     True





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       super().setUp()

       UpperCamelCase			:      Optional[Any] 			=     sp.SentencePieceProcessor()
       spm_model.Load(snake_case_     )
       UpperCamelCase			:      Any 			=     ["<s>", "<pad>", "</s>", "<unk>"]

       vocab += [spm_model.IdToPiece(id_     ) for id_ in range(len(snake_case_     )     )]
       UpperCamelCase			:      Union[str, Any] 			=     dict(zip(snake_case_ ,							range(len(snake_case_     )     )     )     )

       UpperCamelCase			:      Dict 			=     Path(self.tmpdirname     )
       save_json(snake_case_ ,							save_dir / VOCAB_FILES_NAMES["vocab_file"]     )
       if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
        copyfile(snake_case_ ,							save_dir / VOCAB_FILES_NAMES["spm_file"]     )

       UpperCamelCase			:      List[Any] 			=     SpeechaTextTokenizer.from_pretrained(self.tmpdirname     )
       tokenizer.save_pretrained(self.tmpdirname     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     "<pad>"
       UpperCamelCase			:      Optional[Any] 			=     1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_     ) ,							snake_case_     )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_     ) ,							snake_case_     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      int 			=     list(self.get_tokenizer().get_vocab().keys()     )

       self.assertEqual(vocab_keys[0] ,							"<s>"     )
       self.assertEqual(vocab_keys[1] ,							"<pad>"     )
       self.assertEqual(vocab_keys[-1] ,							"j"     )
       self.assertEqual(len(snake_case_     ) ,							1001     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       self.assertEqual(self.get_tokenizer().vocab_size ,							1001     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      List[str] 			=     SpeechaTextTokenizer.from_pretrained(self.tmpdirname     )

       UpperCamelCase			:      Optional[Any] 			=     tokenizer.tokenize("This is a test"     )
       self.assertListEqual(snake_case_ ,							["▁This", "▁is", "▁a", "▁t", "est"]     )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(snake_case_     ) ,							[289, 50, 14, 174, 386] ,							)

       UpperCamelCase			:      Optional[int] 			=     tokenizer.tokenize("I was born in 92000, and this is falsé."     )
       self.assertListEqual(
           snake_case_ ,							[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."] ,							)
       UpperCamelCase			:      Optional[Any] 			=     tokenizer.convert_tokens_to_ids(snake_case_     )
       self.assertListEqual(snake_case_ ,							[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]     )

       UpperCamelCase			:      List[Any] 			=     tokenizer.convert_ids_to_tokens(snake_case_     )
       self.assertListEqual(
           snake_case_ ,							[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."] ,							)





      @slow
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     {"input_ids": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=snake_case_ ,							model_name="facebook/s2t-small-mustc-en-de-st" ,							revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad" ,							)



@require_sentencepiece
class       A__     ( unittest.TestCase    ):
      _UpperCAmelCase			:Tuple							=     """valhalla/s2t_mustc_multilinguial_medium"""

      _UpperCAmelCase			:Union[str, Any]							=     """C'est trop cool"""
      _UpperCAmelCase			:Any							=     """Esto es genial"""





      @classmethod
      def  __UpperCamelCase(				cls     ):
       '''simple docstring'''




       UpperCamelCase			:      SpeechaTextTokenizer 			=     SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name     )
       return cls





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       self.assertEqual(self.tokenizer.lang_code_to_id["pt"] ,							4     )
       self.assertEqual(self.tokenizer.lang_code_to_id["ru"] ,							6     )
       self.assertEqual(self.tokenizer.lang_code_to_id["it"] ,							9     )
       self.assertEqual(self.tokenizer.lang_code_to_id["de"] ,							11     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       self.assertEqual(self.tokenizer.vocab_size ,							1_0000     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       self.assertIn(snake_case_ ,							self.tokenizer.all_special_ids     )
       UpperCamelCase			:      Optional[int] 			=     [ES_CODE, 4, 1601, 47, 7647, 2]
       UpperCamelCase			:      Tuple 			=     self.tokenizer.decode(snake_case_ ,							skip_special_tokens=snake_case_     )
       UpperCamelCase			:      int 			=     self.tokenizer.decode(generated_ids[1:] ,							skip_special_tokens=snake_case_     )
       self.assertEqual(snake_case_ ,							snake_case_     )
       self.assertNotIn(self.tokenizer.eos_token ,							snake_case_     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     "fr"
       UpperCamelCase			:      int 			=     self.tokenizer(self.french_text     ).input_ids
       self.assertEqual(encoded[0] ,							snake_case_     )
       self.assertEqual(encoded[-1] ,							self.tokenizer.eos_token_id     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     "fr"
       self.assertListEqual(self.tokenizer.prefix_tokens ,							[FR_CODE]     )

       UpperCamelCase			:      List[Any] 			=     "es"
       self.assertListEqual(self.tokenizer.prefix_tokens ,							[ES_CODE]     )







import math

import tensorflow as tf
from packaging import version



def 	A_       (  _lowerCAmelCase  )		->				Any:
 UpperCamelCase			:      List[Any] 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      Any 			=     0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0  )							,     x.dtype  )  ))

 return x * cdf



def 	A_       (  _lowerCAmelCase  )		->				Dict:
 UpperCamelCase			:      Union[str, Any] 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      List[Any] 			=     tf.cast(math.pi							,     x.dtype  )
 UpperCamelCase			:      Optional[Any] 			=     tf.cast(0.044_715							,     x.dtype  )
 UpperCamelCase			:      int 			=     0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi  ) * (x + coeff * tf.pow(_lowerCAmelCase							,     3  ))  ))

 return x * cdf



def 	A_       (  _lowerCAmelCase  )		->				List[Any]:
 UpperCamelCase			:      str 			=     tf.convert_to_tensor(_lowerCAmelCase  )

 return x * tf.tanh(tf.math.softplus(_lowerCAmelCase  )  )



def 	A_       (  _lowerCAmelCase  )		->				List[Any]:
 UpperCamelCase			:      Tuple 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      List[Any] 			=     tf.cast(0.044_715							,     x.dtype  )
 UpperCamelCase			:      Optional[Any] 			=     tf.cast(0.7_978_845_608							,     x.dtype  )

 return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)  ))



def 	A_       (  _lowerCAmelCase  )		->				Optional[Any]:
 UpperCamelCase			:      Any 			=     tf.convert_to_tensor(_lowerCAmelCase  )
 UpperCamelCase			:      List[Any] 			=     tf.cast(1.702							,     x.dtype  )
 return x * tf.math.sigmoid(coeff * x  )



def 	A_       (  _lowerCAmelCase  )		->				List[Any]:
 return tf.clip_by_value(_gelu(_lowerCAmelCase  )							,     -10							,     10  )



def 	A_       (  _lowerCAmelCase							,     _lowerCAmelCase=-1  )		->				str:
 UpperCamelCase			,					UpperCamelCase			:      List[Any] 			=     tf.split(_lowerCAmelCase							,     2							,     axis=_lowerCAmelCase  )
 return a * tf.math.sigmoid(_lowerCAmelCase  )


if version.parse(tf.version.VERSION) >= version.parse("""2.4"""):



    def 	A_       (  _lowerCAmelCase  )		->				Any:
     return tf.keras.activations.gelu(_lowerCAmelCase							,     approximate=_lowerCAmelCase  )

    __lowerCamelCase :  Optional[int]										=  tf.keras.activations.gelu


    __lowerCamelCase :  int										=  approximate_gelu_wrap
else:
    __lowerCamelCase :  List[Any]										=  _gelu
    __lowerCamelCase :  Optional[Any]										=  _gelu_new


__lowerCamelCase :  Any										=  {
    """gelu""": gelu,
    """gelu_10""": gelu_aa,
    """gelu_fast""": gelu_fast,
    """gelu_new""": gelu_new,
    """glu""": glu,
    """mish""": mish,
    """quick_gelu""": quick_gelu,
    """relu""": tf.keras.activations.relu,
    """sigmoid""": tf.keras.activations.sigmoid,
    """silu""": tf.keras.activations.swish,
    """swish""": tf.keras.activations.swish,
    """tanh""": tf.keras.activations.tanh,
}



def 	A_       (  _lowerCAmelCase  )		->				Optional[Any]:
 if activation_string in ACTaFN:
  return ACTaFN[activation_string]
 else:
  raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()  )}"""  )
