


import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase         =			get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
   import sentencepiece as sp


__lowerCamelCase         =			5
__lowerCamelCase         =			10


@require_sentencepiece
@require_tokenizers
class   snake_case_ (_snake_case , unittest.TestCase  ):




       """simple docstring"""






       _lowerCamelCase			    =				SpeechaTextTokenizer
       _lowerCamelCase			    =				False
       _lowerCamelCase			    =				True



       def  A_   (     self):

             """simple docstring"""





             super().setUp()

             UpperCAmelCase_  :	Optional[Any]				      =		sp.SentencePieceProcessor()
             spm_model.Load(snake_case_)
             UpperCAmelCase_  :	Any				      =		["<s>", "<pad>", "</s>", "<unk>"]

             vocab += [spm_model.IdToPiece(id_) for id_ in range(len(snake_case_))]
             UpperCAmelCase_  :	Union[str, Any]				      =		dict(zip(snake_case_     ,range(len(snake_case_))))

             UpperCAmelCase_  :	Dict				      =		Path(self.tmpdirname)
             save_json(snake_case_     ,save_dir / VOCAB_FILES_NAMES["vocab_file"])
             if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                   copyfile(snake_case_     ,save_dir / VOCAB_FILES_NAMES["spm_file"])

             UpperCAmelCase_  :	List[Any]				      =		SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
             tokenizer.save_pretrained(self.tmpdirname)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Tuple				      =		"<pad>"
             UpperCAmelCase_  :	Optional[Any]				      =		1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_)     ,snake_case_)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_)     ,snake_case_)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		list(self.get_tokenizer().get_vocab().keys())

             self.assertEqual(vocab_keys[0]     ,"<s>")
             self.assertEqual(vocab_keys[1]     ,"<pad>")
             self.assertEqual(vocab_keys[-1]     ,"j")
             self.assertEqual(len(snake_case_)     ,1001)



       def  A_   (     self):

             """simple docstring"""





             self.assertEqual(self.get_tokenizer().vocab_size     ,1001)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	List[str]				      =		SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

             UpperCAmelCase_  :	Optional[Any]				      =		tokenizer.tokenize("This is a test")
             self.assertListEqual(snake_case_     ,["▁This", "▁is", "▁a", "▁t", "est"])

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(snake_case_)     ,[289, 50, 14, 174, 386]     ,)

             UpperCAmelCase_  :	Optional[int]				      =		tokenizer.tokenize("I was born in 92000, and this is falsé.")
             self.assertListEqual(
                 snake_case_     ,[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]     ,)
             UpperCAmelCase_  :	Optional[Any]				      =		tokenizer.convert_tokens_to_ids(snake_case_)
             self.assertListEqual(snake_case_     ,[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8])

             UpperCAmelCase_  :	List[Any]				      =		tokenizer.convert_ids_to_tokens(snake_case_)
             self.assertListEqual(
                 snake_case_     ,[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]     ,)



       @slow
       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Tuple				      =		{"input_ids": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=snake_case_     ,model_name="facebook/s2t-small-mustc-en-de-st"     ,revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad"     ,)


@require_sentencepiece
class   snake_case_ (unittest.TestCase  ):




       """simple docstring"""






       _lowerCamelCase			    =				"""valhalla/s2t_mustc_multilinguial_medium"""

       _lowerCamelCase			    =				"""C'est trop cool"""
       _lowerCamelCase			    =				"""Esto es genial"""



       @classmethod
       def  A_   (     cls):

             """simple docstring"""





             UpperCAmelCase_  :	SpeechaTextTokenizer				      =		SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
             return cls



       def  A_   (     self):

             """simple docstring"""





             self.assertEqual(self.tokenizer.lang_code_to_id["pt"]     ,4)
             self.assertEqual(self.tokenizer.lang_code_to_id["ru"]     ,6)
             self.assertEqual(self.tokenizer.lang_code_to_id["it"]     ,9)
             self.assertEqual(self.tokenizer.lang_code_to_id["de"]     ,11)



       def  A_   (     self):

             """simple docstring"""





             self.assertEqual(self.tokenizer.vocab_size     ,10000)



       def  A_   (     self):

             """simple docstring"""





             self.assertIn(snake_case_     ,self.tokenizer.all_special_ids)
             UpperCAmelCase_  :	Optional[int]				      =		[ES_CODE, 4, 1601, 47, 7647, 2]
             UpperCAmelCase_  :	Tuple				      =		self.tokenizer.decode(snake_case_     ,skip_special_tokens=snake_case_)
             UpperCAmelCase_  :	int				      =		self.tokenizer.decode(generated_ids[1:]     ,skip_special_tokens=snake_case_)
             self.assertEqual(snake_case_     ,snake_case_)
             self.assertNotIn(self.tokenizer.eos_token     ,snake_case_)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	str				      =		"fr"
             UpperCAmelCase_  :	int				      =		self.tokenizer(self.french_text).input_ids
             self.assertEqual(encoded[0]     ,snake_case_)
             self.assertEqual(encoded[-1]     ,self.tokenizer.eos_token_id)



       def  A_   (     self):

             """simple docstring"""





             UpperCAmelCase_  :	Tuple				      =		"fr"
             self.assertListEqual(self.tokenizer.prefix_tokens     ,[FR_CODE])

             UpperCAmelCase_  :	List[Any]				      =		"es"
             self.assertListEqual(self.tokenizer.prefix_tokens     ,[ES_CODE])





import argparse

import torch

from transformers import FunnelBaseModel, FunnelConfig, FunnelModel, load_tf_weights_in_funnel
from transformers.utils import logging


logging.set_verbosity_info()







def    _snake_case (       __snake_case					,  __snake_case					,  __snake_case					,  __snake_case    )     ->     Optional[Any]:






      '''simple docstring'''







      UpperCAmelCase_  :	List[str]				      =		FunnelConfig.from_json_file(__snake_case    )
      print(F"""Building PyTorch model from configuration: {config}"""    )
      UpperCAmelCase_  :	str				      =		FunnelBaseModel(__snake_case    ) if base_model else FunnelModel(__snake_case    )

      # Load weights from tf checkpoint
      load_tf_weights_in_funnel(__snake_case					,  __snake_case					,  __snake_case    )

      # Save pytorch-model
      print(F"""Save PyTorch model to {pytorch_dump_path}"""    )
      torch.save(model.state_dict()					,  __snake_case    )


if __name__ == "__main__":
   __lowerCamelCase         =			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
   )
   parser.add_argument(
       '''--config_file''',
       default=None,
       type=str,
       required=True,
       help='''The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.''',
   )
   parser.add_argument(
       '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
   )
   parser.add_argument(
       '''--base_model''', action='''store_true''', help='''Whether you want just the base model (no decoder) or not.'''
   )
   __lowerCamelCase         =			parser.parse_args()
   convert_tf_checkpoint_to_pytorch(
       args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path, args.base_model
   )
