


"""simple docstring"""

import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__  							=get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")





@require_sentencepiece
@require_tokenizers
class    lowerCamelCase__						(		_SCREAMING_SNAKE_CASE			,   unittest.TestCase    ):
      a :     List[Any]											=		SpeechTaTokenizer
      a :     Union[str, Any]											=		False
      a :     Any											=		True
      def SCREAMING_SNAKE_CASE_	(		self	:	Any							):






       '''simple docstring'''


       super().setUp()

       # We have a SentencePiece fixture for testing
       __lowercase									=	SpeechTaTokenizer(A_							)

       __lowercase									=	AddedToken("""<mask>"""      ,       lstrip=A_      ,       rstrip=A_							)
       __lowercase									=	mask_token
       tokenizer.add_special_tokens({"""mask_token""": mask_token}							)
       tokenizer.add_tokens(["""<ctc_blank>"""]							)

       tokenizer.save_pretrained(self.tmpdirname							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	"""this is a test"""
       __lowercase									=	"""this is a test"""
       return input_text, output_text
      def SCREAMING_SNAKE_CASE_	(		self	:	int      ,       A_	:	List[Any]      ,       A_	:	List[str]=False      ,       A_	:	int=2_0      ,       A_	:	str=5							):






       '''simple docstring'''


       __lowercase							,		__lowercase									=	self.get_input_output_texts(A_							)
       __lowercase									=	tokenizer.encode(A_      ,       add_special_tokens=A_							)
       __lowercase									=	tokenizer.decode(A_      ,       clean_up_tokenization_spaces=A_							)
       return text, ids
      def SCREAMING_SNAKE_CASE_	(		self	:	Union[str, Any]							):






       '''simple docstring'''


       __lowercase									=	"""<pad>"""
       __lowercase									=	1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_							)      ,       A_							)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_							)      ,       A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       __lowercase									=	list(self.get_tokenizer().get_vocab().keys()							)

       self.assertEqual(vocab_keys[0]      ,       """<s>"""							)
       self.assertEqual(vocab_keys[1]      ,       """<pad>"""							)
       self.assertEqual(vocab_keys[-4]      ,       """œ"""							)
       self.assertEqual(vocab_keys[-2]      ,       """<mask>"""							)
       self.assertEqual(vocab_keys[-1]      ,       """<ctc_blank>"""							)
       self.assertEqual(len(A_							)      ,       8_1							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Union[str, Any]							):






       '''simple docstring'''


       self.assertEqual(self.get_tokenizer().vocab_size      ,       7_9							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	self.get_tokenizers(do_lower_case=A_							)
       for tokenizer in tokenizers:
        with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
         __lowercase									=	tokenizer.vocab_size
         __lowercase									=	len(A_							)

         self.assertNotEqual(A_      ,       0							)

         # We usually have added tokens from the start in tests because our vocab fixtures are
         # smaller than the original vocabs - let's not assert this
         # self.assertEqual(vocab_size, all_size)

         __lowercase									=	["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
         __lowercase									=	tokenizer.add_tokens(A_							)
         __lowercase									=	tokenizer.vocab_size
         __lowercase									=	len(A_							)

         self.assertNotEqual(A_      ,       0							)
         self.assertEqual(A_      ,       A_							)
         self.assertEqual(A_      ,       len(A_							)							)
         self.assertEqual(A_      ,       all_size + len(A_							)							)

         __lowercase									=	tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l"""      ,       add_special_tokens=A_							)

         self.assertGreaterEqual(len(A_							)      ,       4							)
         self.assertGreater(tokens[0]      ,       tokenizer.vocab_size - 1							)
         self.assertGreater(tokens[-3]      ,       tokenizer.vocab_size - 1							)

         __lowercase									=	{"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
         __lowercase									=	tokenizer.add_special_tokens(A_							)
         __lowercase									=	tokenizer.vocab_size
         __lowercase									=	len(A_							)

         self.assertNotEqual(A_      ,       0							)
         self.assertEqual(A_      ,       A_							)
         self.assertEqual(A_      ,       len(A_							)							)
         self.assertEqual(A_      ,       all_size_a + len(A_							)							)

         __lowercase									=	tokenizer.encode(
             """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"""      ,       add_special_tokens=A_							)

         self.assertGreaterEqual(len(A_							)      ,       6							)
         self.assertGreater(tokens[0]      ,       tokenizer.vocab_size - 1							)
         self.assertGreater(tokens[0]      ,       tokens[1]							)
         self.assertGreater(tokens[-3]      ,       tokenizer.vocab_size - 1							)
         self.assertGreater(tokens[-3]      ,       tokens[-4]							)
         self.assertEqual(tokens[0]      ,       tokenizer.eos_token_id							)
         self.assertEqual(tokens[-3]      ,       tokenizer.pad_token_id							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       pass
      def SCREAMING_SNAKE_CASE_	(		self	:	Dict							):






       '''simple docstring'''


       pass
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	self.get_tokenizer()

       __lowercase									=	tokenizer.tokenize("""This is a test"""							)
       # fmt: off
       self.assertListEqual(A_      ,       [SPIECE_UNDERLINE, """T""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """a""", SPIECE_UNDERLINE, """t""", """e""", """s""", """t"""]							)
       # fmt: on

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(A_							)      ,       [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]      ,       )

       __lowercase									=	tokenizer.tokenize("""I was born in 92000, and this is falsé."""							)
       self.assertListEqual(
           A_      ,       [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """92000""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]							)

       __lowercase									=	tokenizer.convert_tokens_to_ids(A_							)
       # fmt: off
       self.assertListEqual(A_      ,       [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]							)
       # fmt: on

       __lowercase									=	tokenizer.convert_ids_to_tokens(A_							)
       self.assertListEqual(
           A_      ,       [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """<unk>""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]							)






      @slow
      def SCREAMING_SNAKE_CASE_	(		self	:	str							):






       '''simple docstring'''


       __lowercase									=	[
           """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
           """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
           """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
           """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
           """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
           """conditioning on both left and right context in all layers.""",
           """The quick brown fox jumps over the lazy dog.""",
       ]

       # fmt: off
       __lowercase									=	{
           """input_ids""": [
               [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
               [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
           ],
           """attention_mask""": [
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
           ]
       }
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=A_      ,       model_name="""microsoft/speecht5_asr"""      ,       revision="""c5ef64c71905caeccde0e4462ef3f9077224c524"""      ,       sequences=A_      ,       )




"""simple docstring"""

import os

import numpy
import onnx


def lowerCAmelCase_      (				UpperCamelCase__	: Optional[int]				,   UpperCamelCase__	: List[str] ):






 """simple docstring"""







 __lowercase									=	a.name
 __lowercase									=	b.name

 __lowercase									=	""""""
 __lowercase									=	""""""

 __lowercase									=	a == b

 __lowercase									=	name_a
 __lowercase									=	name_b

 return res


def lowerCAmelCase_      (				UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: str				,   UpperCamelCase__	: Dict ):






 """simple docstring"""







 for i, input_name in enumerate(node_proto.input ):
  if input_name == name:
   node_proto.input.insert(UpperCamelCase__				,   UpperCamelCase__ )
   node_proto.input.pop(i + 1 )

 if node_proto.op_type == "If":
  _graph_replace_input_with(node_proto.attribute[0].g				,   UpperCamelCase__				,   UpperCamelCase__ )
  _graph_replace_input_with(node_proto.attribute[1].g				,   UpperCamelCase__				,   UpperCamelCase__ )
 if node_proto.op_type == "Loop":
  _graph_replace_input_with(node_proto.attribute[0].g				,   UpperCamelCase__				,   UpperCamelCase__ )


def lowerCAmelCase_      (				UpperCamelCase__	: Optional[Any]				,   UpperCamelCase__	: Optional[int]				,   UpperCamelCase__	: str ):






 """simple docstring"""







 for n in graph_proto.node:
  _node_replace_input_with(UpperCamelCase__				,   UpperCamelCase__				,   UpperCamelCase__ )


def lowerCAmelCase_      (				UpperCamelCase__	: Tuple				,   UpperCamelCase__	: str				,   UpperCamelCase__	: Any ):






 """simple docstring"""







 __lowercase									=	list(model.graph.initializer )
 __lowercase									=	list(model_without_ext.graph.initializer )
 for i, ref_i in ind_to_replace:
  assert inits_with_data[i].name == inits[i].name
  assert inits_with_data[ref_i].name == inits[ref_i].name
  assert i > ref_i

  __lowercase									=	inits[i].name
  __lowercase									=	inits[ref_i].name

  model_without_ext.graph.initializer.remove(inits[i] )

  # for n in model.graph.node:
  _graph_replace_input_with(model_without_ext.graph				,   UpperCamelCase__				,   UpperCamelCase__ )


def lowerCAmelCase_      (				UpperCamelCase__	: List[Any] ):






 """simple docstring"""







 __lowercase									=	os.path.dirname(UpperCamelCase__ )
 __lowercase									=	os.path.basename(UpperCamelCase__ )

 __lowercase									=	onnx.load(os.path.join(UpperCamelCase__				,   UpperCamelCase__ ) )

 __lowercase									=	list(model.graph.initializer )

 __lowercase									=	set()
 __lowercase									=	{}
 __lowercase									=	[]

 __lowercase									=	0

 for i in range(len(UpperCamelCase__ ) ):
  if i in dup_set:
   continue

  for j in range(i + 1				,   len(UpperCamelCase__ ) ):
   if j in dup_set:
    continue
   if _is_equal_tensor_proto(inits[i]				,   inits[j] ):
    dup_set.add(UpperCamelCase__ )
    dup_set.add(UpperCamelCase__ )

    __lowercase									=	inits[j].data_type
    __lowercase									=	numpy.prod(inits[j].dims )
    if dtype == 1:
     mem_size *= 4
    elif dtype == 6:
     mem_size *= 4
    elif dtype == 7 or dtype == 11:
     mem_size *= 8
    else:
     print("""unexpected data type: """				,   UpperCamelCase__ )
    total_reduced_size += mem_size

    __lowercase									=	inits[i].name
    __lowercase									=	inits[j].name

    if name_i in dup_map:
     dup_map[name_i].append(UpperCamelCase__ )
    else:
     __lowercase									=	[name_j]
    ind_to_replace.append((j, i) )

 print("""total reduced size: """				,   total_reduced_size / 1024 / 1024 / 1024				,   """GB""" )

 __lowercase									=	sorted(UpperCamelCase__ )
 _remove_dup_initializers_from_model(UpperCamelCase__				,   UpperCamelCase__				,   UpperCamelCase__ )

 __lowercase									=	"""optimized_""" + model_file_name
 __lowercase									=	os.path.join(UpperCamelCase__				,   UpperCamelCase__ )
 onnx.save(UpperCamelCase__				,   UpperCamelCase__ )

 return new_model
