



import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class      __lowerCAmelCase     (		UpperCAmelCase_							,     unittest.TestCase):
					"""simple docstring"""






					A__				:		str			       =    ProphetNetTokenizer
					A__				:		List[str]			       =    False



					def     _a							(     self					:       Dict					):






								"""simple docstring"""


								super().setUp()

								A__       =    [
								    '[UNK]',
								    '[CLS]',
								    '[SEP]',
								    '[PAD]',
								    '[MASK]',
								    'want',
								    '##want',
								    '##ed',
								    'wa',
								    'un',
								    'runn',
								    '##ing',
								    ',',
								    'low',
								    'lowest',
								]
								A__       =    os.path.join(self.tmpdirname				,					VOCAB_FILES_NAMES['vocab_file']					)
								with open(self.vocab_file				,					'w'				,					encoding='utf-8'					) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)



					def     _a							(     self					:       Any				,					_snake_case					:       Union[str, Any]					):






								"""simple docstring"""


								A__       =    'UNwant\u00E9d,running'
								A__       =    'unwanted, running'
								return input_text, output_text



					def     _a							(     self					:       List[Any]					):






								"""simple docstring"""


								A__       =    self.tokenizer_class(self.vocab_file					)

								A__       =    tokenizer.tokenize('UNwant\u00E9d,running'					)
								self.assertListEqual(_snake_case				,					['un', '##want', '##ed', ',', 'runn', '##ing']					)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case					)				,					[9, 6, 7, 12, 10, 11]					)



					def     _a							(     self					:       List[str]					):






								"""simple docstring"""


								A__       =    BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)				,					['ah', '\u535A', '\u63A8', 'zz']					)



					def     _a							(     self					:       Dict					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)				,					['hello', '!', 'how', 'are', 'you', '?']					)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)				,					['hello']					)



					def     _a							(     self					:       List[Any]					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case				,					strip_accents=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)				,					['hällo', '!', 'how', 'are', 'you', '?']					)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)				,					['h\u00E9llo']					)



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case				,					strip_accents=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)				,					['hallo', '!', 'how', 'are', 'you', '?']					)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)				,					['hello']					)



					def     _a							(     self					:       Tuple					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)				,					['hallo', '!', 'how', 'are', 'you', '?']					)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)				,					['hello']					)



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)				,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)



					def     _a							(     self					:       str					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case				,					strip_accents=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)				,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)



					def     _a							(     self					:       Tuple					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case				,					strip_accents=_snake_case					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)				,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)



					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    BasicTokenizer(do_lower_case=_snake_case				,					never_split=['[UNK]']					)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)				,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)



					def     _a							(     self					:       Optional[int]					):






								"""simple docstring"""


								A__       =    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

								A__       =    {}
								for i, token in enumerate(_snake_case					):
											A__       =    i
								A__       =    WordpieceTokenizer(vocab=_snake_case				,					unk_token='[UNK]'					)

								self.assertListEqual(tokenizer.tokenize(''					)				,					[]					)

								self.assertListEqual(tokenizer.tokenize('unwanted running'					)				,					['un', '##want', '##ed', 'runn', '##ing']					)

								self.assertListEqual(tokenizer.tokenize('unwantedX running'					)				,					['[UNK]', 'runn', '##ing']					)



					@require_torch
					def     _a							(     self					:       List[str]					):






								"""simple docstring"""


								A__       =    self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'					)

								A__       =    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
								A__       =    [10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
								A__       =    tokenizer(_snake_case				,					padding=_snake_case				,					return_tensors='pt'					)
								self.assertIsInstance(_snake_case				,					_snake_case					)
								A__       =    list(batch.input_ids.numpy()[0]					)
								self.assertListEqual(_snake_case				,					_snake_case					)

								self.assertEqual((2, 9)				,					batch.input_ids.shape					)
								self.assertEqual((2, 9)				,					batch.attention_mask.shape					)



					def     _a							(     self					:       Union[str, Any]					):






								"""simple docstring"""


								self.assertTrue(_is_whitespace(' '					)					)
								self.assertTrue(_is_whitespace('\t'					)					)
								self.assertTrue(_is_whitespace('\r'					)					)
								self.assertTrue(_is_whitespace('\n'					)					)
								self.assertTrue(_is_whitespace('\u00A0'					)					)

								self.assertFalse(_is_whitespace('A'					)					)
								self.assertFalse(_is_whitespace('-'					)					)



					def     _a							(     self					:       str					):






								"""simple docstring"""


								self.assertTrue(_is_control('\u0005'					)					)

								self.assertFalse(_is_control('A'					)					)
								self.assertFalse(_is_control(' '					)					)
								self.assertFalse(_is_control('\t'					)					)
								self.assertFalse(_is_control('\r'					)					)



					def     _a							(     self					:       List[str]					):






								"""simple docstring"""


								self.assertTrue(_is_punctuation('-'					)					)
								self.assertTrue(_is_punctuation('$'					)					)
								self.assertTrue(_is_punctuation('`'					)					)
								self.assertTrue(_is_punctuation('.'					)					)

								self.assertFalse(_is_punctuation('A'					)					)
								self.assertFalse(_is_punctuation(' '					)					)



					@slow
					def     _a							(     self					:       Any					):






								"""simple docstring"""


								A__       =    self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'					)

								A__       =    tokenizer.encode('sequence builders'				,					add_special_tokens=_snake_case					)
								A__       =    tokenizer.encode('multi-sequence build'				,					add_special_tokens=_snake_case					)

								A__       =    tokenizer.build_inputs_with_special_tokens(_snake_case					)
								A__       =    tokenizer.build_inputs_with_special_tokens(_snake_case				,					_snake_case					)

								assert encoded_sentence == text + [1_02]
								assert encoded_pair == text + [1_02] + text_a + [1_02]









import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
							import optax
							from flax.training.common_utils import onehot

							from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
							from transformers.models.ta.modeling_flax_ta import shift_tokens_right


@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class      __lowerCAmelCase     (		unittest.TestCase      ):
					"""simple docstring"""






					@slow
					def     _a							(     self					:       int					):






								"""simple docstring"""


								A__       =    FlaxMTaForConditionalGeneration.from_pretrained('google/mt5-small'					)
								A__       =    AutoTokenizer.from_pretrained('google/mt5-small'					)

								A__       =    tokenizer('Hello there'				,					return_tensors='np'					).input_ids
								A__       =    tokenizer('Hi I am'				,					return_tensors='np'					).input_ids

								A__       =    shift_tokens_right(_snake_case				,					model.config.pad_token_id				,					model.config.decoder_start_token_id					)

								A__       =    model(_snake_case				,					decoder_input_ids=_snake_case					).logits
								A__       =    optax.softmax_cross_entropy(_snake_case				,					onehot(_snake_case				,					logits.shape[-1]					)					).mean()

								A__       =    -(labels.shape[-1] * loss.item())

								A__       =    -84.9127
								self.assertTrue(abs(mtf_score - EXPECTED_SCORE					) < 1E-4					)




