



'''simple docstring'''



import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_     :       Any								=						get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')
@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_							(					_UpperCamelCase    ,				unittest.TestCase			):
						_lowerCAmelCase     :   str	     =				SpeechTaTokenizer
						_lowerCAmelCase     :   Tuple	     =				False
						_lowerCAmelCase     :   Optional[Any]	     =				True







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													super().setUp()

													# We have a SentencePiece fixture for testing
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			SpeechTaTokenizer(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			AddedToken('''<mask>'''				,    lstrip=lowerCAmelCase__				,    rstrip=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			mask_token
													tokenizer.add_special_tokens({'''mask_token''': mask_token}				)
													tokenizer.add_tokens(['''<ctc_blank>''']				)

													tokenizer.save_pretrained(self.tmpdirname				)







						def 					__lowercase    (  self   :      List[Any]				,    lowerCAmelCase__   :      Tuple				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			"this is a test"
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			"this is a test"
													return input_text, output_text







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      List[Any]=False				,    lowerCAmelCase__   :      Optional[Any]=20				,    lowerCAmelCase__   :      Tuple=5				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_input_output_texts(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.decode(lowerCAmelCase__				,    clean_up_tokenization_spaces=lowerCAmelCase__				)
													return text, ids







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			"<pad>"
													SCREAMING_SNAKE_CASE					:      Tuple     					=			1

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__				)				,    lowerCAmelCase__				)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__				)				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			list(self.get_tokenizer().get_vocab().keys()				)

													self.assertEqual(vocab_keys[0]				,    '''<s>'''				)
													self.assertEqual(vocab_keys[1]				,    '''<pad>'''				)
													self.assertEqual(vocab_keys[-4]				,    '''œ'''				)
													self.assertEqual(vocab_keys[-2]				,    '''<mask>'''				)
													self.assertEqual(vocab_keys[-1]				,    '''<ctc_blank>'''				)
													self.assertEqual(len(lowerCAmelCase__				)				,    81				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													self.assertEqual(self.get_tokenizer().vocab_size				,    79				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			self.get_tokenizers(do_lower_case=lowerCAmelCase__				)
													for tokenizer in tokenizers:
																				with self.subTest(F"""{tokenizer.__class__.__name__}"""				):
																											SCREAMING_SNAKE_CASE					:      int     					=			tokenizer.vocab_size
																											SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			len(lowerCAmelCase__				)

																											self.assertNotEqual(lowerCAmelCase__				,    0				)

																											# We usually have added tokens from the start in tests because our vocab fixtures are
																											# smaller than the original vocabs - let's not assert this
																											# self.assertEqual(vocab_size, all_size)

																											SCREAMING_SNAKE_CASE					:      Dict     					=			["aaaaa bbbbbb", "cccccccccdddddddd"]
																											SCREAMING_SNAKE_CASE					:      Dict     					=			tokenizer.add_tokens(lowerCAmelCase__				)
																											SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.vocab_size
																											SCREAMING_SNAKE_CASE					:      List[str]     					=			len(lowerCAmelCase__				)

																											self.assertNotEqual(lowerCAmelCase__				,    0				)
																											self.assertEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
																											self.assertEqual(lowerCAmelCase__				,    len(lowerCAmelCase__				)				)
																											self.assertEqual(lowerCAmelCase__				,    all_size + len(lowerCAmelCase__				)				)

																											SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''				,    add_special_tokens=lowerCAmelCase__				)

																											self.assertGreaterEqual(len(lowerCAmelCase__				)				,    4				)
																											self.assertGreater(tokens[0]				,    tokenizer.vocab_size - 1				)
																											self.assertGreater(tokens[-3]				,    tokenizer.vocab_size - 1				)

																											SCREAMING_SNAKE_CASE					:      Optional[int]     					=			{"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
																											SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.add_special_tokens(lowerCAmelCase__				)
																											SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.vocab_size
																											SCREAMING_SNAKE_CASE					:      Optional[int]     					=			len(lowerCAmelCase__				)

																											self.assertNotEqual(lowerCAmelCase__				,    0				)
																											self.assertEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
																											self.assertEqual(lowerCAmelCase__				,    len(lowerCAmelCase__				)				)
																											self.assertEqual(lowerCAmelCase__				,    all_size_a + len(lowerCAmelCase__				)				)

																											SCREAMING_SNAKE_CASE					:      Dict     					=			tokenizer.encode(
																											    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''				,    add_special_tokens=lowerCAmelCase__				)

																											self.assertGreaterEqual(len(lowerCAmelCase__				)				,    6				)
																											self.assertGreater(tokens[0]				,    tokenizer.vocab_size - 1				)
																											self.assertGreater(tokens[0]				,    tokens[1]				)
																											self.assertGreater(tokens[-3]				,    tokenizer.vocab_size - 1				)
																											self.assertGreater(tokens[-3]				,    tokens[-4]				)
																											self.assertEqual(tokens[0]				,    tokenizer.eos_token_id				)
																											self.assertEqual(tokens[-3]				,    tokenizer.pad_token_id				)







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			self.get_tokenizer()

													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.tokenize('''This is a test'''				)
													# fmt: off
													self.assertListEqual(lowerCAmelCase__				,    [SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t''']				)
													# fmt: on

													self.assertListEqual(
													    tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)				,    [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]				,    )

													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
													self.assertListEqual(
													    lowerCAmelCase__				,    [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)
													# fmt: off
													self.assertListEqual(lowerCAmelCase__				,    [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]				)
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(
													    lowerCAmelCase__				,    [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']				)







						@slow
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													# Use custom sequence because this tokenizer does not handle numbers.
													SCREAMING_SNAKE_CASE					:      List[str]     					=			[
													    "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
													    "general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
													    "Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained "
													    "models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.",
													    "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
													    "conditioning on both left and right context in all layers.",
													    "The quick brown fox jumps over the lazy dog.",
													]

													# fmt: off
													SCREAMING_SNAKE_CASE					:      Dict     					=			{
													    "input_ids": [
													        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
													        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
													        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
													    ],
													    "attention_mask": [
													        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
													        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													    ]
													}
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=lowerCAmelCase__				,    model_name='''microsoft/speecht5_asr'''				,    revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''				,    sequences=lowerCAmelCase__				,    )









'''simple docstring'''



import os


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_doctest_list.py
lowerCAmelCase_     :       Optional[Any]								=						'.'


if __name__ == "__main__":
							lowerCAmelCase_     :       Optional[Any]								=						os.path.join(REPO_PATH, 'utils/documentation_tests.txt')
							lowerCAmelCase_     :       Dict								=						[]
							lowerCAmelCase_     :       Any								=						[]
							with open(doctest_file_path) as fp:
														for line in fp:
																					lowerCAmelCase_     :       List[str]								=						line.strip()
																					lowerCAmelCase_     :       Optional[Any]								=						os.path.join(REPO_PATH, line)
																					if not (os.path.isfile(path) or os.path.isdir(path)):
																												non_existent_paths.append(line)
																					all_paths.append(path)
							if len(non_existent_paths) > 0:
														lowerCAmelCase_     :       Dict								=						'\n'.join(non_existent_paths)
														raise ValueError(f'`utils/documentation_tests.txt` contains non-existent paths:\n{non_existent_paths}')
							if all_paths != sorted(all_paths):
														raise ValueError('Files in `utils/documentation_tests.txt` are not in alphabetical order.')




