



"""simple docstring"""





import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      =		get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")
@require_sentencepiece
@require_tokenizers
class    lowercase__     (					SCREAMING_SNAKE_CASE_       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  PegasusTokenizer
       UpperCamelCase		=  PegasusTokenizerFast
       UpperCamelCase		=  True
       UpperCamelCase		=  True



       def   lowercase__       (							self    :						List[str]					)				->		Any:


             '''simple docstring'''




             super().setUp()

             # We have a SentencePiece fixture for testing
             UpperCAmelCase_    				=							PegasusTokenizer(UpperCamelCase__					)
             tokenizer.save_pretrained(self.tmpdirname					)



       @cached_property
       def   lowercase__       (							self    :						List[Any]					)				->		Dict:


             '''simple docstring'''




             return PegasusTokenizer.from_pretrained("google/pegasus-large"					)



       def   lowercase__       (							self    :						int	,       **_UpperCAmelCase    :						Optional[Any]					)				->		Optional[int]:


             '''simple docstring'''




             return PegasusTokenizer.from_pretrained(self.tmpdirname	,       **UpperCamelCase__					)



       def   lowercase__       (							self    :						str	,       _UpperCAmelCase    :						Union[str, Any]					)				->		List[Any]:


             '''simple docstring'''




             return ("This is a test", "This is a test")



       def   lowercase__       (							self    :						Tuple					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							'''</s>'''
             UpperCAmelCase_    				=							1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__					)	,       UpperCamelCase__					)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__					)	,       UpperCamelCase__					)



       def   lowercase__       (							self    :						Optional[Any]					)				->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							list(self.get_tokenizer().get_vocab().keys()					)

             self.assertEqual(vocab_keys[0]	,       "<pad>"					)
             self.assertEqual(vocab_keys[1]	,       "</s>"					)
             self.assertEqual(vocab_keys[-1]	,       "v"					)
             self.assertEqual(len(UpperCamelCase__					)	,       1103					)



       def   lowercase__       (							self    :						Union[str, Any]					)				->		List[str]:


             '''simple docstring'''




             self.assertEqual(self.get_tokenizer().vocab_size	,       1103					)



       def   lowercase__       (							self    :						Tuple					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.rust_tokenizer_class.from_pretrained(self.tmpdirname					)
             UpperCAmelCase_    				=							self.tokenizer_class.from_pretrained(self.tmpdirname					)
             UpperCAmelCase_    				=							(
                 '''Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'''
                 ''' </s> <pad> <pad> <pad>'''
             )
             UpperCAmelCase_    				=							rust_tokenizer([raw_input_str]	,       return_tensors=UpperCamelCase__	,       add_special_tokens=UpperCamelCase__					).input_ids[0]
             UpperCAmelCase_    				=							py_tokenizer([raw_input_str]	,       return_tensors=UpperCamelCase__	,       add_special_tokens=UpperCamelCase__					).input_ids[0]
             self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__					)



       def   lowercase__       (							self    :						Optional[int]					)				->		Any:


             '''simple docstring'''




             UpperCAmelCase_    				=							self._large_tokenizer
             # <mask_1> masks whole sentence while <mask_2> masks single word
             UpperCAmelCase_    				=							'''<mask_1> To ensure a <mask_2> flow of bank resolutions.'''
             UpperCAmelCase_    				=							[2, 413, 615, 114, 3, 1971, 113, 1679, 10710, 107, 1]
             UpperCAmelCase_    				=							tokenizer([raw_input_str]	,       return_tensors=UpperCamelCase__					).input_ids[0]
             self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__					)



       def   lowercase__       (							self    :						Any					)				->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self._large_tokenizer
             # The tracebacks for the following asserts are **better** without messages or self.assertEqual
             assert tokenizer.vocab_size == 96103
             assert tokenizer.pad_token_id == 0
             assert tokenizer.eos_token_id == 1
             assert tokenizer.offset == 103
             assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
             assert tokenizer.unk_token == "<unk>"
             assert tokenizer.model_max_length == 1024
             UpperCAmelCase_    				=							'''To ensure a smooth flow of bank resolutions.'''
             UpperCAmelCase_    				=							[413, 615, 114, 2291, 1971, 113, 1679, 10710, 107, 1]
             UpperCAmelCase_    				=							tokenizer([raw_input_str]	,       return_tensors=UpperCamelCase__					).input_ids[0]
             self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__					)
             assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]					) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]



       @require_torch
       def   lowercase__       (							self    :						List[str]					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							['''This is going to be way too long.''' * 150, '''short example''']
             UpperCAmelCase_    				=							['''not super long but more than 5 tokens''', '''tiny''']
             UpperCAmelCase_    				=							self._large_tokenizer(UpperCamelCase__	,       padding=UpperCamelCase__	,       truncation=UpperCamelCase__	,       return_tensors="pt"					)
             UpperCAmelCase_    				=							self._large_tokenizer(
                 text_target=UpperCamelCase__	,       max_length=5	,       padding=UpperCamelCase__	,       truncation=UpperCamelCase__	,       return_tensors="pt"					)

             assert batch.input_ids.shape == (2, 1024)
             assert batch.attention_mask.shape == (2, 1024)
             assert targets["input_ids"].shape == (2, 5)
             assert len(UpperCamelCase__					) == 2  # input_ids, attention_mask.







       @slow
       def   lowercase__       (							self    :						int					)				->		Dict:


             '''simple docstring'''




             UpperCAmelCase_    				=							{'''input_ids''': [[38979, 143, 18485, 606, 130, 26669, 87686, 121, 54189, 1129, 111, 26669, 87686, 121, 9114, 14787, 121, 13249, 158, 592, 956, 121, 14621, 31576, 143, 62613, 108, 9688, 930, 43430, 11562, 62613, 304, 108, 11443, 897, 108, 9314, 17415, 63399, 108, 11443, 7614, 18316, 118, 4284, 7148, 12430, 143, 1400, 25703, 158, 111, 4284, 7148, 11772, 143, 21297, 1064, 158, 122, 204, 3506, 1754, 1133, 14787, 1581, 115, 33224, 4482, 111, 1355, 110, 29173, 317, 50833, 108, 20147, 94665, 111, 77198, 107, 1], [110, 62613, 117, 638, 112, 1133, 121, 20098, 1355, 79050, 13872, 135, 1596, 53541, 1352, 141, 13039, 5542, 124, 302, 518, 111, 268, 2956, 115, 149, 4427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1235, 2799, 18289, 17780, 204, 109, 9474, 1296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=UpperCamelCase__	,       model_name="google/bigbird-pegasus-large-arxiv"	,       revision="ba85d0851d708441f91440d509690f1ab6353415"	,       )



@require_sentencepiece
@require_tokenizers
class    lowercase__     (					SCREAMING_SNAKE_CASE_       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  PegasusTokenizer
       UpperCamelCase		=  PegasusTokenizerFast
       UpperCamelCase		=  True
       UpperCamelCase		=  True



       def   lowercase__       (							self    :						Optional[Any]					)				->		Tuple:


             '''simple docstring'''




             super().setUp()

             # We have a SentencePiece fixture for testing
             UpperCAmelCase_    				=							PegasusTokenizer(UpperCamelCase__	,       offset=0	,       mask_token_sent=UpperCamelCase__	,       mask_token="[MASK]"					)
             tokenizer.save_pretrained(self.tmpdirname					)



       @cached_property
       def   lowercase__       (							self    :						int					)				->		Optional[int]:


             '''simple docstring'''




             return PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv"					)



       def   lowercase__       (							self    :						Any	,       **_UpperCAmelCase    :						int					)				->		Any:


             '''simple docstring'''




             return PegasusTokenizer.from_pretrained(self.tmpdirname	,       **UpperCamelCase__					)



       def   lowercase__       (							self    :						Optional[Any]	,       _UpperCAmelCase    :						Tuple					)				->		int:


             '''simple docstring'''




             return ("This is a test", "This is a test")



       def   lowercase__       (							self    :						Optional[int]					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.rust_tokenizer_class.from_pretrained(self.tmpdirname					)
             UpperCAmelCase_    				=							self.tokenizer_class.from_pretrained(self.tmpdirname					)
             UpperCAmelCase_    				=							(
                 '''Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'''
                 ''' <pad> <pad> <pad>'''
             )
             UpperCAmelCase_    				=							rust_tokenizer([raw_input_str]	,       return_tensors=UpperCamelCase__	,       add_special_tokens=UpperCamelCase__					).input_ids[0]
             UpperCAmelCase_    				=							py_tokenizer([raw_input_str]	,       return_tensors=UpperCamelCase__	,       add_special_tokens=UpperCamelCase__					).input_ids[0]
             self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__					)



       @require_torch
       def   lowercase__       (							self    :						List[str]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							['''This is going to be way too long.''' * 1000, '''short example''']
             UpperCAmelCase_    				=							['''not super long but more than 5 tokens''', '''tiny''']
             UpperCAmelCase_    				=							self._large_tokenizer(UpperCamelCase__	,       padding=UpperCamelCase__	,       truncation=UpperCamelCase__	,       return_tensors="pt"					)
             UpperCAmelCase_    				=							self._large_tokenizer(
                 text_target=UpperCamelCase__	,       max_length=5	,       padding=UpperCamelCase__	,       truncation=UpperCamelCase__	,       return_tensors="pt"					)

             assert batch.input_ids.shape == (2, 4096)
             assert batch.attention_mask.shape == (2, 4096)
             assert targets["input_ids"].shape == (2, 5)
             assert len(UpperCamelCase__					) == 2  # input_ids, attention_mask.







       def   lowercase__       (							self    :						str					)				->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							(
                 '''This is an example string that is used to test the original TF implementation against the HF'''
                 ''' implementation'''
             )

             UpperCAmelCase_    				=							self._large_tokenizer(UpperCamelCase__					).input_ids

             self.assertListEqual(
                 UpperCamelCase__	,       [182, 117, 142, 587, 4211, 120, 117, 263, 112, 804, 109, 856, 25016, 3137, 464, 109, 26955, 3137, 1]	,       )









"""simple docstring"""





import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      =		get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
class    lowercase__     (					SCREAMING_SNAKE_CASE       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  XLMProphetNetTokenizer
       UpperCamelCase		=  False
       UpperCamelCase		=  True



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Any:


             '''simple docstring'''




             super().setUp()

             # We have a SentencePiece fixture for testing
             UpperCAmelCase_    				=							XLMProphetNetTokenizer(_UpperCAmelCase	,       keep_accents=_UpperCAmelCase					)
             tokenizer.save_pretrained(self.tmpdirname					)



       def   lowercase__       (							self    :						Tuple					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							"[PAD]"
             UpperCAmelCase_    				=							0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase					)	,       _UpperCAmelCase					)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase					)	,       _UpperCAmelCase					)



       def   lowercase__       (							self    :						Dict					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							list(self.get_tokenizer().get_vocab().keys()					)

             self.assertEqual(vocab_keys[0]	,       "[PAD]"					)
             self.assertEqual(vocab_keys[1]	,       "[CLS]"					)
             self.assertEqual(vocab_keys[-1]	,       "j"					)
             self.assertEqual(len(_UpperCAmelCase					)	,       1012					)



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             self.assertEqual(self.get_tokenizer().vocab_size	,       1012					)



       def   lowercase__       (							self    :						Tuple					)				->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							XLMProphetNetTokenizer(_UpperCAmelCase	,       keep_accents=_UpperCAmelCase					)

             UpperCAmelCase_    				=							tokenizer.tokenize("This is a test"					)
             self.assertListEqual(_UpperCAmelCase	,       ["▁This", "▁is", "▁a", "▁t", "est"]					)

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)	,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,       )

             UpperCAmelCase_    				=							tokenizer.tokenize("I was born in 92000, and this is falsé."					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "9",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "é",
                     ".",
                 ]	,       )
             UpperCAmelCase_    				=							tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
                 ]	,       )

             UpperCAmelCase_    				=							tokenizer.convert_ids_to_tokens(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "[UNK]",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "[UNK]",
                     ".",
                 ]	,       )



       @cached_property
       def   lowercase__       (							self    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"					)



       @slow
       def   lowercase__       (							self    :						List[Any]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							"Hello World!"
             UpperCAmelCase_    				=							[35389, 6672, 49, 2]
             self.assertListEqual(_UpperCAmelCase	,       self.big_tokenizer.encode(_UpperCAmelCase					)					)







       @slow
       def   lowercase__       (							self    :						Optional[int]					)				->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							{"input_ids": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=_UpperCAmelCase	,       model_name="microsoft/xprophetnet-large-wiki100-cased"	,       revision="1acad1643ddd54a44df6a1b797ada8373685d90e"	,       )




