





'''simple docstring'''



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				Dict											=       MobileBertTokenizer
	a    :				str											=       MobileBertTokenizerFast
	a    :				Tuple											=       True
	a    :				Any											=       True
	a    :				Union[str, Any]											=       filter_non_english
	a    :				Optional[int]											=       "google/mobilebert-uncased"


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				super().setUp()

				__lowercase										=							[
				    '''[UNK]''',
				    '''[CLS]''',
				    '''[SEP]''',
				    '''[PAD]''',
				    '''[MASK]''',
				    '''want''',
				    '''##want''',
				    '''##ed''',
				    '''wa''',
				    '''un''',
				    '''runn''',
				    '''##ing''',
				    ''',''',
				    '''low''',
				    '''lowest''',
				]
				__lowercase										=							os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['''vocab_file'''] )
				with open(self.vocab_file     ,'''w'''     ,encoding='''utf-8''' ) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )

				__lowercase										=							[
				    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
				    for tokenizer_def in self.tokenizers_list
				]


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							int:







				'''simple docstring'''





				__lowercase										=							'''UNwant\u00E9d,running'''
				__lowercase										=							'''unwanted, running'''
				return input_text, output_text


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							self.tokenizer_class(self.vocab_file )

				__lowercase										=							tokenizer.tokenize('''UNwant\u00E9d,running''' )
				self.assertListEqual(_lowerCamelCase     ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase )     ,[9, 6, 7, 12, 10, 11] )


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				if not self.test_rust_tokenizer:
							return

				__lowercase										=							self.get_tokenizer()
				__lowercase										=							self.get_rust_tokenizer()

				__lowercase										=							'''UNwant\u00E9d,running'''

				__lowercase										=							tokenizer.tokenize(_lowerCamelCase )
				__lowercase										=							rust_tokenizer.tokenize(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							tokenizer.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )
				__lowercase										=							rust_tokenizer.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							self.get_rust_tokenizer()
				__lowercase										=							tokenizer.encode(_lowerCamelCase )
				__lowercase										=							rust_tokenizer.encode(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

				# With lower casing
				__lowercase										=							self.get_tokenizer(do_lower_case=_lowerCamelCase )
				__lowercase										=							self.get_rust_tokenizer(do_lower_case=_lowerCamelCase )

				__lowercase										=							'''UNwant\u00E9d,running'''

				__lowercase										=							tokenizer.tokenize(_lowerCamelCase )
				__lowercase										=							rust_tokenizer.tokenize(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							tokenizer.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )
				__lowercase										=							rust_tokenizer.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							self.get_rust_tokenizer()
				__lowercase										=							tokenizer.encode(_lowerCamelCase )
				__lowercase										=							rust_tokenizer.encode(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )     ,['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )


	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )     ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,['''hello'''] )


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase     ,strip_accents=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,['''h\u00E9llo'''] )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase     ,strip_accents=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,['''hello'''] )


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,['''hello'''] )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )     ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase     ,strip_accents=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase     ,strip_accents=_lowerCamelCase )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							BasicTokenizer(do_lower_case=_lowerCamelCase     ,never_split=['''[UNK]'''] )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )     ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )


	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				__lowercase										=							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

				__lowercase										=							{}
				for i, token in enumerate(_lowerCamelCase ):
							__lowercase										=							i
				__lowercase										=							WordpieceTokenizer(vocab=_lowerCamelCase     ,unk_token='''[UNK]''' )

				self.assertListEqual(tokenizer.tokenize('''''' )     ,[] )

				self.assertListEqual(tokenizer.tokenize('''unwanted running''' )     ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

				self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )     ,['''[UNK]''', '''runn''', '''##ing'''] )


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				self.assertTrue(_is_whitespace(''' ''' ) )
				self.assertTrue(_is_whitespace('''\t''' ) )
				self.assertTrue(_is_whitespace('''\r''' ) )
				self.assertTrue(_is_whitespace('''\n''' ) )
				self.assertTrue(_is_whitespace('''\u00A0''' ) )

				self.assertFalse(_is_whitespace('''A''' ) )
				self.assertFalse(_is_whitespace('''-''' ) )


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				self.assertTrue(_is_control('''\u0005''' ) )

				self.assertFalse(_is_control('''A''' ) )
				self.assertFalse(_is_control(''' ''' ) )
				self.assertFalse(_is_control('''\t''' ) )
				self.assertFalse(_is_control('''\r''' ) )


	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				self.assertTrue(_is_punctuation('''-''' ) )
				self.assertTrue(_is_punctuation('''$''' ) )
				self.assertTrue(_is_punctuation('''`''' ) )
				self.assertTrue(_is_punctuation('''.''' ) )

				self.assertFalse(_is_punctuation('''A''' ) )
				self.assertFalse(_is_punctuation(''' ''' ) )


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							self.get_tokenizer()
				__lowercase										=							self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_lowerCamelCase ) for t in ['''Test''', '''\xad''', '''test''']]     ,[['''[UNK]'''], [], ['''[UNK]''']] )

				self.assertListEqual(
				    [rust_tokenizer.tokenize(_lowerCamelCase ) for t in ['''Test''', '''\xad''', '''test''']]     ,[['''[UNK]'''], [], ['''[UNK]''']] )


	@slow
	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							self.tokenizer_class.from_pretrained('''google/mobilebert-uncased''' )

				__lowercase										=							tokenizer.encode('''sequence builders'''     ,add_special_tokens=_lowerCamelCase )
				__lowercase										=							tokenizer.encode('''multi-sequence build'''     ,add_special_tokens=_lowerCamelCase )

				__lowercase										=							tokenizer.build_inputs_with_special_tokens(_lowerCamelCase )
				__lowercase										=							tokenizer.build_inputs_with_special_tokens(_lowerCamelCase     ,_lowerCamelCase )

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
										__lowercase										=							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase )

										__lowercase										=							f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowercase										=							tokenizer_r.encode_plus(
										    _lowerCamelCase     ,return_attention_mask=_lowerCamelCase     ,return_token_type_ids=_lowerCamelCase     ,return_offsets_mapping=_lowerCamelCase     ,add_special_tokens=_lowerCamelCase     ,)

										__lowercase										=							tokenizer_r.do_lower_case if hasattr(_lowerCamelCase     ,'''do_lower_case''' ) else False
										__lowercase										=							(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), '''A'''),
										        ((1, 2), ''','''),
										        ((3, 5), '''na'''),
										        ((5, 6), '''##ï'''),
										        ((6, 8), '''##ve'''),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), '''Allen'''),
										        ((21, 23), '''##NL'''),
										        ((23, 24), '''##P'''),
										        ((25, 33), '''sentence'''),
										        ((33, 34), '''.'''),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), '''a'''),
										        ((1, 2), ''','''),
										        ((3, 8), '''naive'''),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), '''allen'''),
										        ((21, 23), '''##nl'''),
										        ((23, 24), '''##p'''),
										        ((25, 33), '''sentence'''),
										        ((33, 34), '''.'''),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
										self.assertEqual([e[0] for e in expected_results]     ,tokens['''offset_mapping'''] )


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							['''的''', '''人''', '''有''']
				__lowercase										=							''''''.join(_lowerCamelCase )
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
										__lowercase										=							True
										__lowercase										=							self.tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase )
										__lowercase										=							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase )

										__lowercase										=							tokenizer_p.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )
										__lowercase										=							tokenizer_r.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )

										__lowercase										=							tokenizer_r.convert_ids_to_tokens(_lowerCamelCase )
										__lowercase										=							tokenizer_p.convert_ids_to_tokens(_lowerCamelCase )

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )
										self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )

										__lowercase										=							False
										__lowercase										=							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase )
										__lowercase										=							self.tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase )

										__lowercase										=							tokenizer_r.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )
										__lowercase										=							tokenizer_p.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase )

										__lowercase										=							tokenizer_r.convert_ids_to_tokens(_lowerCamelCase )
										__lowercase										=							tokenizer_p.convert_ids_to_tokens(_lowerCamelCase )

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowercase										=							[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase )
										]
										self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )
										self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )










'''simple docstring'''



from typing import List, Optional, Union

import numpy as np
import PIL.Image

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


_SCREAMING_SNAKE_CASE					=    logging.get_logger(__name__)






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	a    :				int											=       ["pixel_values"]


	def __init__(self     ,_lowerCamelCase = True     ,_lowerCamelCase = 32     ,_lowerCamelCase=PILImageResampling.BILINEAR     ,_lowerCamelCase = True     ,**_lowerCamelCase     ,)							->							None:







				'''simple docstring'''





				__lowercase										=							do_resize
				__lowercase										=							do_rescale
				__lowercase										=							size_divisor
				__lowercase										=							resample
				super().__init__(**_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase = None     ,**_lowerCamelCase )							->							np.ndarray:







				'''simple docstring'''





				__lowercase						,		__lowercase										=							get_image_size(_lowerCamelCase )
				# Rounds the height and width down to the closest multiple of size_divisor
				__lowercase										=							height // size_divisor * size_divisor
				__lowercase										=							width // size_divisor * size_divisor
				__lowercase										=							resize(_lowerCamelCase     ,(new_h, new_w)     ,resample=_lowerCamelCase     ,data_format=_lowerCamelCase     ,**_lowerCamelCase )
				return image


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase = None     ,**_lowerCamelCase )							->							np.ndarray:







				'''simple docstring'''





				return rescale(image=_lowerCamelCase     ,scale=_lowerCamelCase     ,data_format=_lowerCamelCase     ,**_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None     ,_lowerCamelCase = None     ,_lowerCamelCase=None     ,_lowerCamelCase = None     ,_lowerCamelCase = None     ,_lowerCamelCase = ChannelDimension.FIRST     ,**_lowerCamelCase     ,)							->							BatchFeature:







				'''simple docstring'''





				__lowercase										=							do_resize if do_resize is not None else self.do_resize
				__lowercase										=							do_rescale if do_rescale is not None else self.do_rescale
				__lowercase										=							size_divisor if size_divisor is not None else self.size_divisor
				__lowercase										=							resample if resample is not None else self.resample

				if do_resize and size_divisor is None:
							raise ValueError('''size_divisor is required for resizing''' )

				__lowercase										=							make_list_of_images(_lowerCamelCase )

				if not valid_images(_lowerCamelCase ):
							raise ValueError('''Invalid image(s)''' )

				# All transformations expect numpy arrays.
				__lowercase										=							[to_numpy_array(_lowerCamelCase ) for img in images]

				if do_resize:
							__lowercase										=							[self.resize(_lowerCamelCase     ,size_divisor=_lowerCamelCase     ,resample=_lowerCamelCase ) for image in images]

				if do_rescale:
							__lowercase										=							[self.rescale(_lowerCamelCase     ,scale=1 / 255 ) for image in images]

				__lowercase										=							[to_channel_dimension_format(_lowerCamelCase     ,_lowerCamelCase ) for image in images]

				__lowercase										=							{'''pixel_values''': images}
				return BatchFeature(data=_lowerCamelCase     ,tensor_type=_lowerCamelCase )





