






'''simple docstring'''






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class a_    (    UpperCamelCase_		,      unittest.TestCase							):
			_snake_case        =							MobileBertTokenizer
			_snake_case        =							MobileBertTokenizerFast
			_snake_case        =							True
			_snake_case        =							True
			_snake_case        =							filter_non_english
			_snake_case        =							"""google/mobilebert-uncased"""







			def     SCREAMING_SNAKE_CASE__					(self)			->      str:
						"""simple docstring"""






						super().setUp()

						__snake_case				:      str     							= [
						    '[UNK]',
						    '[CLS]',
						    '[SEP]',
						    '[PAD]',
						    '[MASK]',
						    'want',
						    '##want',
						    '##ed',
						    'wa',
						    'un',
						    'runn',
						    '##ing',
						    ',',
						    'low',
						    'lowest',
						]
						__snake_case				:      Tuple     							= os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file'])
						with open(self.vocab_file					,   'w'					,   encoding='utf-8') as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

						__snake_case				:      str     							= [
						    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
						    for tokenizer_def in self.tokenizers_list
						]







			def     SCREAMING_SNAKE_CASE__					(self					,   __a)			->      Tuple:
						"""simple docstring"""






						__snake_case				:      Any     							= 'UNwant\u00E9d,running'
						__snake_case				:      Union[str, Any]     							= 'unwanted, running'
						return input_text, output_text







			def     SCREAMING_SNAKE_CASE__					(self)			->      Any:
						"""simple docstring"""






						__snake_case				:      Dict     							= self.tokenizer_class(self.vocab_file)

						__snake_case				:      int     							= tokenizer.tokenize('UNwant\u00E9d,running')
						self.assertListEqual(__a					,   ['un', '##want', '##ed', ',', 'runn', '##ing'])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)					,   [9, 6, 7, 1_2, 1_0, 1_1])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Optional[int]:
						"""simple docstring"""






						if not self.test_rust_tokenizer:
									return

						__snake_case				:      Dict     							= self.get_tokenizer()
						__snake_case				:      Optional[Any]     							= self.get_rust_tokenizer()

						__snake_case				:      int     							= 'UNwant\u00E9d,running'

						__snake_case				:      Tuple     							= tokenizer.tokenize(__a)
						__snake_case				:      int     							= rust_tokenizer.tokenize(__a)
						self.assertListEqual(__a					,   __a)

						__snake_case				:      Union[str, Any]     							= tokenizer.encode(__a					,   add_special_tokens=__a)
						__snake_case				:      int     							= rust_tokenizer.encode(__a					,   add_special_tokens=__a)
						self.assertListEqual(__a					,   __a)

						__snake_case				:      Optional[int]     							= self.get_rust_tokenizer()
						__snake_case				:      List[str]     							= tokenizer.encode(__a)
						__snake_case				:      int     							= rust_tokenizer.encode(__a)
						self.assertListEqual(__a					,   __a)

						# With lower casing
						__snake_case				:      Tuple     							= self.get_tokenizer(do_lower_case=__a)
						__snake_case				:      int     							= self.get_rust_tokenizer(do_lower_case=__a)

						__snake_case				:      Optional[int]     							= 'UNwant\u00E9d,running'

						__snake_case				:      str     							= tokenizer.tokenize(__a)
						__snake_case				:      Any     							= rust_tokenizer.tokenize(__a)
						self.assertListEqual(__a					,   __a)

						__snake_case				:      List[str]     							= tokenizer.encode(__a					,   add_special_tokens=__a)
						__snake_case				:      Dict     							= rust_tokenizer.encode(__a					,   add_special_tokens=__a)
						self.assertListEqual(__a					,   __a)

						__snake_case				:      Optional[Any]     							= self.get_rust_tokenizer()
						__snake_case				:      Union[str, Any]     							= tokenizer.encode(__a)
						__snake_case				:      Optional[Any]     							= rust_tokenizer.encode(__a)
						self.assertListEqual(__a					,   __a)







			def     SCREAMING_SNAKE_CASE__					(self)			->      Tuple:
						"""simple docstring"""






						__snake_case				:      str     							= BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')					,   ['ah', '\u535A', '\u63A8', 'zz'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      List[Any]:
						"""simple docstring"""






						__snake_case				:      Dict     							= BasicTokenizer(do_lower_case=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')					,   ['hello', '!', 'how', 'are', 'you', '?'])
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo')					,   ['hello'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Union[str, Any]:
						"""simple docstring"""






						__snake_case				:      Dict     							= BasicTokenizer(do_lower_case=__a					,   strip_accents=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')					,   ['hällo', '!', 'how', 'are', 'you', '?'])
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo')					,   ['h\u00E9llo'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Optional[Any]:
						"""simple docstring"""






						__snake_case				:      int     							= BasicTokenizer(do_lower_case=__a					,   strip_accents=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')					,   ['hallo', '!', 'how', 'are', 'you', '?'])
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo')					,   ['hello'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Any:
						"""simple docstring"""






						__snake_case				:      Union[str, Any]     							= BasicTokenizer(do_lower_case=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')					,   ['hallo', '!', 'how', 'are', 'you', '?'])
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo')					,   ['hello'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Dict:
						"""simple docstring"""






						__snake_case				:      int     							= BasicTokenizer(do_lower_case=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')					,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Optional[int]:
						"""simple docstring"""






						__snake_case				:      Optional[int]     							= BasicTokenizer(do_lower_case=__a					,   strip_accents=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')					,   ['HäLLo', '!', 'how', 'Are', 'yoU', '?'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Any:
						"""simple docstring"""






						__snake_case				:      int     							= BasicTokenizer(do_lower_case=__a					,   strip_accents=__a)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')					,   ['HaLLo', '!', 'how', 'Are', 'yoU', '?'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Any:
						"""simple docstring"""






						__snake_case				:      Optional[Any]     							= BasicTokenizer(do_lower_case=__a					,   never_split=['[UNK]'])

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')					,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Any:
						"""simple docstring"""






						__snake_case				:      Optional[int]     							= ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

						__snake_case				:      Optional[Any]     							= {}
						for i, token in enumerate(__a):
									__snake_case				:      str     							= i
						__snake_case				:      Optional[Any]     							= WordpieceTokenizer(vocab=__a					,   unk_token='[UNK]')

						self.assertListEqual(tokenizer.tokenize('')					,   [])

						self.assertListEqual(tokenizer.tokenize('unwanted running')					,   ['un', '##want', '##ed', 'runn', '##ing'])

						self.assertListEqual(tokenizer.tokenize('unwantedX running')					,   ['[UNK]', 'runn', '##ing'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Union[str, Any]:
						"""simple docstring"""






						self.assertTrue(_is_whitespace(' '))
						self.assertTrue(_is_whitespace('\t'))
						self.assertTrue(_is_whitespace('\r'))
						self.assertTrue(_is_whitespace('\n'))
						self.assertTrue(_is_whitespace('\u00A0'))

						self.assertFalse(_is_whitespace('A'))
						self.assertFalse(_is_whitespace('-'))







			def     SCREAMING_SNAKE_CASE__					(self)			->      List[Any]:
						"""simple docstring"""






						self.assertTrue(_is_control('\u0005'))

						self.assertFalse(_is_control('A'))
						self.assertFalse(_is_control(' '))
						self.assertFalse(_is_control('\t'))
						self.assertFalse(_is_control('\r'))







			def     SCREAMING_SNAKE_CASE__					(self)			->      Tuple:
						"""simple docstring"""






						self.assertTrue(_is_punctuation('-'))
						self.assertTrue(_is_punctuation('$'))
						self.assertTrue(_is_punctuation('`'))
						self.assertTrue(_is_punctuation('.'))

						self.assertFalse(_is_punctuation('A'))
						self.assertFalse(_is_punctuation(' '))







			def     SCREAMING_SNAKE_CASE__					(self)			->      Dict:
						"""simple docstring"""






						__snake_case				:      Any     							= self.get_tokenizer()
						__snake_case				:      Union[str, Any]     							= self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]					,   [['[UNK]'], [], ['[UNK]']])

						self.assertListEqual(
						    [rust_tokenizer.tokenize(__a) for t in ['Test', '\xad', 'test']]					,   [['[UNK]'], [], ['[UNK]']])







			@slow
			def     SCREAMING_SNAKE_CASE__					(self)			->      Union[str, Any]:
						"""simple docstring"""






						__snake_case				:      int     							= self.tokenizer_class.from_pretrained('google/mobilebert-uncased')

						__snake_case				:      Optional[Any]     							= tokenizer.encode('sequence builders'					,   add_special_tokens=__a)
						__snake_case				:      Optional[int]     							= tokenizer.encode('multi-sequence build'					,   add_special_tokens=__a)

						__snake_case				:      List[Any]     							= tokenizer.build_inputs_with_special_tokens(__a)
						__snake_case				:      str     							= tokenizer.build_inputs_with_special_tokens(__a					,   __a)

						assert encoded_sentence == [1_0_1] + text + [1_0_2]
						assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







			def     SCREAMING_SNAKE_CASE__					(self)			->      str:
						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
												__snake_case				:      Optional[Any]     							= self.rust_tokenizer_class.from_pretrained(__a					,   **__a)

												__snake_case				:      Dict     							= F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
												__snake_case				:      Union[str, Any]     							= tokenizer_r.encode_plus(
												    __a					,   return_attention_mask=__a					,   return_token_type_ids=__a					,   return_offsets_mapping=__a					,   add_special_tokens=__a					,   )

												__snake_case				:      List[Any]     							= tokenizer_r.do_lower_case if hasattr(__a					,   'do_lower_case') else False
												__snake_case				:      Tuple     							= (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'A'),
												        ((1, 2), ','),
												        ((3, 5), 'na'),
												        ((5, 6), '##ï'),
												        ((6, 8), '##ve'),
												        ((9, 1_5), tokenizer_r.mask_token),
												        ((1_6, 2_1), 'Allen'),
												        ((2_1, 2_3), '##NL'),
												        ((2_3, 2_4), '##P'),
												        ((2_5, 3_3), 'sentence'),
												        ((3_3, 3_4), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'a'),
												        ((1, 2), ','),
												        ((3, 8), 'naive'),
												        ((9, 1_5), tokenizer_r.mask_token),
												        ((1_6, 2_1), 'allen'),
												        ((2_1, 2_3), '##nl'),
												        ((2_3, 2_4), '##p'),
												        ((2_5, 3_3), 'sentence'),
												        ((3_3, 3_4), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]					,   tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
												self.assertEqual([e[0] for e in expected_results]					,   tokens['offset_mapping'])







			def     SCREAMING_SNAKE_CASE__					(self)			->      Union[str, Any]:
						"""simple docstring"""






						__snake_case				:      Tuple     							= ['的', '人', '有']
						__snake_case				:      List[str]     							= ''.join(__a)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
												__snake_case				:      Any     							= True
												__snake_case				:      int     							= self.tokenizer_class.from_pretrained(__a					,   **__a)
												__snake_case				:      int     							= self.rust_tokenizer_class.from_pretrained(__a					,   **__a)

												__snake_case				:      List[Any]     							= tokenizer_p.encode(__a					,   add_special_tokens=__a)
												__snake_case				:      Optional[Any]     							= tokenizer_r.encode(__a					,   add_special_tokens=__a)

												__snake_case				:      int     							= tokenizer_r.convert_ids_to_tokens(__a)
												__snake_case				:      List[Any]     							= tokenizer_p.convert_ids_to_tokens(__a)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(__a					,   __a)
												self.assertListEqual(__a					,   __a)

												__snake_case				:      Tuple     							= False
												__snake_case				:      List[Any]     							= self.rust_tokenizer_class.from_pretrained(__a					,   **__a)
												__snake_case				:      Optional[int]     							= self.tokenizer_class.from_pretrained(__a					,   **__a)

												__snake_case				:      str     							= tokenizer_r.encode(__a					,   add_special_tokens=__a)
												__snake_case				:      int     							= tokenizer_p.encode(__a					,   add_special_tokens=__a)

												__snake_case				:      str     							= tokenizer_r.convert_ids_to_tokens(__a)
												__snake_case				:      Any     							= tokenizer_p.convert_ids_to_tokens(__a)

												# it is expected that only the first Chinese character is not preceded by "##".
												__snake_case				:      Optional[Any]     							= [
												    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a)
												]
												self.assertListEqual(__a					,   __a)
												self.assertListEqual(__a					,   __a)







'''simple docstring'''






# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
__A    	=   abspath(join(dirname(dirname(__file__)), '''src'''))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='''ignore''', category=FutureWarning)




def        _SCREAMING_SNAKE_CASE  (				A  :       Tuple  )    -> str:







   """simple docstring"""
   from diffusers.utils.testing_utils import pytest_addoption_shared

   pytest_addoption_shared(A  )





def        _SCREAMING_SNAKE_CASE  (				A  :       int  )    -> Optional[int]:







   """simple docstring"""
   from diffusers.utils.testing_utils import pytest_terminal_summary_main

   __snake_case				:      Any     							= terminalreporter.config.getoption('--make-reports'  )
   if make_reports:
      pytest_terminal_summary_main(A  ,		id=A  )