





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class       __A					(					_lowerCamelCase    ,       unittest.TestCase				):

   '''simple docstring'''







   lowerCAmelCase_							  =		MobileBertTokenizer
   lowerCAmelCase_							  =		MobileBertTokenizerFast
   lowerCAmelCase_							  =		True
   lowerCAmelCase_							  =		True
   lowerCAmelCase_							  =		filter_non_english
   lowerCAmelCase_							  =		"""google/mobilebert-uncased"""





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    super().setUp()

    lowerCamelCase__              =	[
        '''[UNK]''',
        '''[CLS]''',
        '''[SEP]''',
        '''[PAD]''',
        '''[MASK]''',
        '''want''',
        '''##want''',
        '''##ed''',
        '''wa''',
        '''un''',
        '''runn''',
        '''##ing''',
        ''',''',
        '''low''',
        '''lowest''',
    ]
    lowerCamelCase__              =	os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''vocab_file''']							)
    with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8'''							) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)

    lowerCamelCase__              =	[
        (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
        for tokenizer_def in self.tokenizers_list
    ]





   def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



    '''simple docstring'''







    lowerCamelCase__              =	'''UNwant\u00E9d,running'''
    lowerCamelCase__              =	'''unwanted, running'''
    return input_text, output_text





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	self.tokenizer_class(self.vocab_file							)

    lowerCamelCase__              =	tokenizer.tokenize('''UNwant\u00E9d,running'''							)
    self.assertListEqual(A__ ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(A__							) ,						[9, 6, 7, 1_2, 1_0, 1_1]							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    if not self.test_rust_tokenizer:
     return

    lowerCamelCase__              =	self.get_tokenizer()
    lowerCamelCase__              =	self.get_rust_tokenizer()

    lowerCamelCase__              =	'''UNwant\u00E9d,running'''

    lowerCamelCase__              =	tokenizer.tokenize(A__							)
    lowerCamelCase__              =	rust_tokenizer.tokenize(A__							)
    self.assertListEqual(A__ ,						A__							)

    lowerCamelCase__              =	tokenizer.encode(A__ ,						add_special_tokens=A__							)
    lowerCamelCase__              =	rust_tokenizer.encode(A__ ,						add_special_tokens=A__							)
    self.assertListEqual(A__ ,						A__							)

    lowerCamelCase__              =	self.get_rust_tokenizer()
    lowerCamelCase__              =	tokenizer.encode(A__							)
    lowerCamelCase__              =	rust_tokenizer.encode(A__							)
    self.assertListEqual(A__ ,						A__							)

    # With lower casing
    lowerCamelCase__              =	self.get_tokenizer(do_lower_case=A__							)
    lowerCamelCase__              =	self.get_rust_tokenizer(do_lower_case=A__							)

    lowerCamelCase__              =	'''UNwant\u00E9d,running'''

    lowerCamelCase__              =	tokenizer.tokenize(A__							)
    lowerCamelCase__              =	rust_tokenizer.tokenize(A__							)
    self.assertListEqual(A__ ,						A__							)

    lowerCamelCase__              =	tokenizer.encode(A__ ,						add_special_tokens=A__							)
    lowerCamelCase__              =	rust_tokenizer.encode(A__ ,						add_special_tokens=A__							)
    self.assertListEqual(A__ ,						A__							)

    lowerCamelCase__              =	self.get_rust_tokenizer()
    lowerCamelCase__              =	tokenizer.encode(A__							)
    lowerCamelCase__              =	rust_tokenizer.encode(A__							)
    self.assertListEqual(A__ ,						A__							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer()

    self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							) ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							) ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							) ,						['''hello''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__ ,						strip_accents=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							) ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							) ,						['''h\u00E9llo''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__ ,						strip_accents=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							) ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							) ,						['''hello''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							) ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							) ,						['''hello''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							) ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__ ,						strip_accents=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							) ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__ ,						strip_accents=A__							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							) ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	BasicTokenizer(do_lower_case=A__ ,						never_split=['''[UNK]''']							)

    self.assertListEqual(
        tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							) ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

    lowerCamelCase__              =	{}
    for i, token in enumerate(A__							):
     lowerCamelCase__              =	i
    lowerCamelCase__              =	WordpieceTokenizer(vocab=A__ ,						unk_token='''[UNK]'''							)

    self.assertListEqual(tokenizer.tokenize(''''''							) ,						[]							)

    self.assertListEqual(tokenizer.tokenize('''unwanted running'''							) ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

    self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							) ,						['''[UNK]''', '''runn''', '''##ing''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    self.assertTrue(_is_whitespace(''' '''							)							)
    self.assertTrue(_is_whitespace('''\t'''							)							)
    self.assertTrue(_is_whitespace('''\r'''							)							)
    self.assertTrue(_is_whitespace('''\n'''							)							)
    self.assertTrue(_is_whitespace('''\u00A0'''							)							)

    self.assertFalse(_is_whitespace('''A'''							)							)
    self.assertFalse(_is_whitespace('''-'''							)							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    self.assertTrue(_is_control('''\u0005'''							)							)

    self.assertFalse(_is_control('''A'''							)							)
    self.assertFalse(_is_control(''' '''							)							)
    self.assertFalse(_is_control('''\t'''							)							)
    self.assertFalse(_is_control('''\r'''							)							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    self.assertTrue(_is_punctuation('''-'''							)							)
    self.assertTrue(_is_punctuation('''$'''							)							)
    self.assertTrue(_is_punctuation('''`'''							)							)
    self.assertTrue(_is_punctuation('''.'''							)							)

    self.assertFalse(_is_punctuation('''A'''							)							)
    self.assertFalse(_is_punctuation(''' '''							)							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	self.get_tokenizer()
    lowerCamelCase__              =	self.get_rust_tokenizer()

    # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
    self.assertListEqual([tokenizer.tokenize(A__							) for t in ['''Test''', '''\xad''', '''test''']] ,						[['''[UNK]'''], [], ['''[UNK]''']]							)

    self.assertListEqual(
        [rust_tokenizer.tokenize(A__							) for t in ['''Test''', '''\xad''', '''test''']] ,						[['''[UNK]'''], [], ['''[UNK]''']]							)





   @slow
   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''							)

    lowerCamelCase__              =	tokenizer.encode('''sequence builders''' ,						add_special_tokens=A__							)
    lowerCamelCase__              =	tokenizer.encode('''multi-sequence build''' ,						add_special_tokens=A__							)

    lowerCamelCase__              =	tokenizer.build_inputs_with_special_tokens(A__							)
    lowerCamelCase__              =	tokenizer.build_inputs_with_special_tokens(A__ ,						A__							)

    assert encoded_sentence == [1_0_1] + text + [1_0_2]
    assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
      lowerCamelCase__              =	self.rust_tokenizer_class.from_pretrained(A__ ,						**A__							)

      lowerCamelCase__              =	F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
      lowerCamelCase__              =	tokenizer_r.encode_plus(
          A__ ,						return_attention_mask=A__ ,						return_token_type_ids=A__ ,						return_offsets_mapping=A__ ,						add_special_tokens=A__ ,						)

      lowerCamelCase__              =	tokenizer_r.do_lower_case if hasattr(A__ ,						'''do_lower_case'''							) else False
      lowerCamelCase__              =	(
          [
              ((0, 0), tokenizer_r.cls_token),
              ((0, 1), '''A'''),
              ((1, 2), ''','''),
              ((3, 5), '''na'''),
              ((5, 6), '''##ï'''),
              ((6, 8), '''##ve'''),
              ((9, 1_5), tokenizer_r.mask_token),
              ((1_6, 2_1), '''Allen'''),
              ((2_1, 2_3), '''##NL'''),
              ((2_3, 2_4), '''##P'''),
              ((2_5, 3_3), '''sentence'''),
              ((3_3, 3_4), '''.'''),
              ((0, 0), tokenizer_r.sep_token),
          ]
          if not do_lower_case
          else [
              ((0, 0), tokenizer_r.cls_token),
              ((0, 1), '''a'''),
              ((1, 2), ''','''),
              ((3, 8), '''naive'''),
              ((9, 1_5), tokenizer_r.mask_token),
              ((1_6, 2_1), '''allen'''),
              ((2_1, 2_3), '''##nl'''),
              ((2_3, 2_4), '''##p'''),
              ((2_5, 3_3), '''sentence'''),
              ((3_3, 3_4), '''.'''),
              ((0, 0), tokenizer_r.sep_token),
          ]
      )

      self.assertEqual(
          [e[1] for e in expected_results] ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
      self.assertEqual([e[0] for e in expected_results] ,						tokens['''offset_mapping''']							)





   def 	__lowerCamelCase       (					self							):



    '''simple docstring'''







    lowerCamelCase__              =	['''的''', '''人''', '''有''']
    lowerCamelCase__              =	''''''.join(A__							)
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
      lowerCamelCase__              =	True
      lowerCamelCase__              =	self.tokenizer_class.from_pretrained(A__ ,						**A__							)
      lowerCamelCase__              =	self.rust_tokenizer_class.from_pretrained(A__ ,						**A__							)

      lowerCamelCase__              =	tokenizer_p.encode(A__ ,						add_special_tokens=A__							)
      lowerCamelCase__              =	tokenizer_r.encode(A__ ,						add_special_tokens=A__							)

      lowerCamelCase__              =	tokenizer_r.convert_ids_to_tokens(A__							)
      lowerCamelCase__              =	tokenizer_p.convert_ids_to_tokens(A__							)

      # it is expected that each Chinese character is not preceded by "##"
      self.assertListEqual(A__ ,						A__							)
      self.assertListEqual(A__ ,						A__							)

      lowerCamelCase__              =	False
      lowerCamelCase__              =	self.rust_tokenizer_class.from_pretrained(A__ ,						**A__							)
      lowerCamelCase__              =	self.tokenizer_class.from_pretrained(A__ ,						**A__							)

      lowerCamelCase__              =	tokenizer_r.encode(A__ ,						add_special_tokens=A__							)
      lowerCamelCase__              =	tokenizer_p.encode(A__ ,						add_special_tokens=A__							)

      lowerCamelCase__              =	tokenizer_r.convert_ids_to_tokens(A__							)
      lowerCamelCase__              =	tokenizer_p.convert_ids_to_tokens(A__							)

      # it is expected that only the first Chinese character is not preceded by "##".
      lowerCamelCase__              =	[
          F'##{token}' if idx != 0 else token for idx, token in enumerate(A__							)
      ]
      self.assertListEqual(A__ ,						A__							)
      self.assertListEqual(A__ ,						A__							)










from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		"""ClapFeatureExtractor"""
			lowerCAmelCase_							  =		("""RobertaTokenizer""", """RobertaTokenizerFast""")





			def __init__(					self ,						__lowerCAmelCase ,						__lowerCAmelCase							):



				'''simple docstring'''







				super().__init__(__lowerCAmelCase ,						__lowerCAmelCase							)





			def __call__(					self ,						__lowerCAmelCase=None ,						__lowerCAmelCase=None ,						__lowerCAmelCase=None ,						**__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	kwargs.pop('''sampling_rate''' ,						__lowerCAmelCase							)

				if text is None and audios is None:
					raise ValueError('''You have to specify either text or audios. Both cannot be none.'''							)

				if text is not None:
					lowerCamelCase__              =	self.tokenizer(__lowerCAmelCase ,						return_tensors=__lowerCAmelCase ,						**__lowerCAmelCase							)

				if audios is not None:
					lowerCamelCase__              =	self.feature_extractor(
					    __lowerCAmelCase ,						sampling_rate=__lowerCAmelCase ,						return_tensors=__lowerCAmelCase ,						**__lowerCAmelCase							)

				if text is not None and audios is not None:
					lowerCamelCase__              =	audio_features.input_features
					return encoding
				elif text is not None:
					return encoding
				else:
					return BatchEncoding(data=dict(**__lowerCAmelCase							) ,						tensor_type=__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						*__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.batch_decode(*__lowerCAmelCase ,						**__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						*__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.decode(*__lowerCAmelCase ,						**__lowerCAmelCase							)





			@property
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer.model_input_names
				lowerCamelCase__              =	self.feature_extractor.model_input_names
				return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names							)							)



