
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class        SCREAMING_SNAKE_CASE     (UpperCAmelCase    ,			unittest.TestCase   ):
   _UpperCamelCase					:	Tuple				      = BertTokenizer
   _UpperCamelCase					:	Tuple				      = BertTokenizerFast
   _UpperCamelCase					:	Dict				      = True
   _UpperCamelCase					:	List[Any]				      = True
   _UpperCamelCase					:	List[Any]				      = filter_non_english


   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[int] )->     Union[str, Any]:





     """simple docstring"""







     super().setUp()

     lowercase__			    =						[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         '[PAD]',
         '[MASK]',
         'want',
         '##want',
         '##ed',
         'wa',
         'un',
         'runn',
         '##ing',
         ',',
         'low',
         'lowest',
     ]
     lowercase__			    =						os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES['vocab_file'] )
     with open(self.vocab_file    ,  'w'    ,  encoding='utf-8' ) as vocab_writer:
       vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )


   def  SCREAMING_SNAKE_CASE_							(					self    :       str    ,  a    :       List[str] )->     Union[str, Any]:





     """simple docstring"""







     lowercase__			    =						'UNwant\u00E9d,running'
     lowercase__			    =						'unwanted, running'
     return input_text, output_text


   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[Any] )->     Optional[int]:





     """simple docstring"""







     lowercase__			    =						self.tokenizer_class(self.vocab_file )

     lowercase__			    =						tokenizer.tokenize('UNwant\u00E9d,running' )
     self.assertListEqual(a    ,  ['un', '##want', '##ed', ',', 'runn', '##ing'] )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(a )    ,  [9, 6, 7, 12, 10, 11] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Any )->     Optional[int]:





     """simple docstring"""







     if not self.test_rust_tokenizer:
       return

     lowercase__			    =						self.get_tokenizer()
     lowercase__			    =						self.get_rust_tokenizer()

     lowercase__			    =						'UNwant\u00E9d,running'

     lowercase__			    =						tokenizer.tokenize(a )
     lowercase__			    =						rust_tokenizer.tokenize(a )
     self.assertListEqual(a    ,  a )

     lowercase__			    =						tokenizer.encode(a    ,  add_special_tokens=a )
     lowercase__			    =						rust_tokenizer.encode(a    ,  add_special_tokens=a )
     self.assertListEqual(a    ,  a )

     lowercase__			    =						self.get_rust_tokenizer()
     lowercase__			    =						tokenizer.encode(a )
     lowercase__			    =						rust_tokenizer.encode(a )
     self.assertListEqual(a    ,  a )

     # With lower casing
     lowercase__			    =						self.get_tokenizer(do_lower_case=a )
     lowercase__			    =						self.get_rust_tokenizer(do_lower_case=a )

     lowercase__			    =						'UNwant\u00E9d,running'

     lowercase__			    =						tokenizer.tokenize(a )
     lowercase__			    =						rust_tokenizer.tokenize(a )
     self.assertListEqual(a    ,  a )

     lowercase__			    =						tokenizer.encode(a    ,  add_special_tokens=a )
     lowercase__			    =						rust_tokenizer.encode(a    ,  add_special_tokens=a )
     self.assertListEqual(a    ,  a )

     lowercase__			    =						self.get_rust_tokenizer()
     lowercase__			    =						tokenizer.encode(a )
     lowercase__			    =						rust_tokenizer.encode(a )
     self.assertListEqual(a    ,  a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       int )->     Optional[int]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )    ,  ['ah', '\u535A', '\u63A8', 'zz'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Any )->     Union[str, Any]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )    ,  ['hello', '!', 'how', 'are', 'you', '?'] )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )    ,  ['hello'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[str] )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a    ,  strip_accents=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )    ,  ['hällo', '!', 'how', 'are', 'you', '?'] )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )    ,  ['h\u00E9llo'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a    ,  strip_accents=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )    ,  ['hallo', '!', 'how', 'are', 'you', '?'] )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )    ,  ['hello'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[Any] )->     Tuple:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )    ,  ['hallo', '!', 'how', 'are', 'you', '?'] )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )    ,  ['hello'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       str )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )    ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[Any] )->     List[Any]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a    ,  strip_accents=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )    ,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[str] )->     List[str]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a    ,  strip_accents=a )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )    ,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       int )->     Dict:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer(do_lower_case=a    ,  never_split=['[UNK]'] )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )    ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Any )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						BasicTokenizer()
     lowercase__			    =						'a\n\'ll !!to?\'d of, can\'t.'
     lowercase__			    =						['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
     self.assertListEqual(tokenizer.tokenize(a )    ,  a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Union[str, Any] )->     Tuple:





     """simple docstring"""







     lowercase__			    =						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

     lowercase__			    =						{}
     for i, token in enumerate(a ):
       lowercase__			    =						i
     lowercase__			    =						WordpieceTokenizer(vocab=a    ,  unk_token='[UNK]' )

     self.assertListEqual(tokenizer.tokenize('' )    ,  [] )

     self.assertListEqual(tokenizer.tokenize('unwanted running' )    ,  ['un', '##want', '##ed', 'runn', '##ing'] )

     self.assertListEqual(tokenizer.tokenize('unwantedX running' )    ,  ['[UNK]', 'runn', '##ing'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Any )->     int:





     """simple docstring"""







     self.assertTrue(_is_whitespace(' ' ) )
     self.assertTrue(_is_whitespace('\t' ) )
     self.assertTrue(_is_whitespace('\r' ) )
     self.assertTrue(_is_whitespace('\n' ) )
     self.assertTrue(_is_whitespace('\u00A0' ) )

     self.assertFalse(_is_whitespace('A' ) )
     self.assertFalse(_is_whitespace('-' ) )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Any )->     Dict:





     """simple docstring"""







     self.assertTrue(_is_control('\u0005' ) )

     self.assertFalse(_is_control('A' ) )
     self.assertFalse(_is_control(' ' ) )
     self.assertFalse(_is_control('\t' ) )
     self.assertFalse(_is_control('\r' ) )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Dict )->     Union[str, Any]:





     """simple docstring"""







     self.assertTrue(_is_punctuation('-' ) )
     self.assertTrue(_is_punctuation('$' ) )
     self.assertTrue(_is_punctuation('`' ) )
     self.assertTrue(_is_punctuation('.' ) )

     self.assertFalse(_is_punctuation('A' ) )
     self.assertFalse(_is_punctuation(' ' ) )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[Any] )->     Union[str, Any]:





     """simple docstring"""







     lowercase__			    =						self.get_tokenizer()
     lowercase__			    =						self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(a ) for t in ['Test', '\xad', 'test']]    ,  [['[UNK]'], [], ['[UNK]']] )

     self.assertListEqual(
         [rust_tokenizer.tokenize(a ) for t in ['Test', '\xad', 'test']]    ,  [['[UNK]'], [], ['[UNK]']] )


   @slow
   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[Any] )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						self.tokenizer_class.from_pretrained('bert-base-uncased' )

     lowercase__			    =						tokenizer.encode('sequence builders'    ,  add_special_tokens=a )
     lowercase__			    =						tokenizer.encode('multi-sequence build'    ,  add_special_tokens=a )

     lowercase__			    =						tokenizer.build_inputs_with_special_tokens(a )
     lowercase__			    =						tokenizer.build_inputs_with_special_tokens(a    ,  a )

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     Any:





     """simple docstring"""







     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
         lowercase__			    =						self.rust_tokenizer_class.from_pretrained(a    ,  **a )

         lowercase__			    =						f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
         lowercase__			    =						tokenizer_r.encode_plus(
             a    ,  return_attention_mask=a    ,  return_token_type_ids=a    ,  return_offsets_mapping=a    ,  add_special_tokens=a    ,  )

         lowercase__			    =						tokenizer_r.do_lower_case if hasattr(a    ,  'do_lower_case' ) else False
         lowercase__			    =						(
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), 'A'),
                 ((1, 2), ','),
                 ((3, 5), 'na'),
                 ((5, 6), '##ï'),
                 ((6, 8), '##ve'),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), 'Allen'),
                 ((21, 23), '##NL'),
                 ((23, 24), '##P'),
                 ((25, 33), 'sentence'),
                 ((33, 34), '.'),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), 'a'),
                 ((1, 2), ','),
                 ((3, 8), 'naive'),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), 'allen'),
                 ((21, 23), '##nl'),
                 ((23, 24), '##p'),
                 ((25, 33), 'sentence'),
                 ((33, 34), '.'),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )

         self.assertEqual(
             [e[1] for e in expected_results]    ,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
         self.assertEqual([e[0] for e in expected_results]    ,  tokens['offset_mapping'] )


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     Dict:





     """simple docstring"""







     lowercase__			    =						['的', '人', '有']
     lowercase__			    =						''.join(a )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
         lowercase__			    =						True
         lowercase__			    =						self.tokenizer_class.from_pretrained(a    ,  **a )
         lowercase__			    =						self.rust_tokenizer_class.from_pretrained(a    ,  **a )

         lowercase__			    =						tokenizer_p.encode(a    ,  add_special_tokens=a )
         lowercase__			    =						tokenizer_r.encode(a    ,  add_special_tokens=a )

         lowercase__			    =						tokenizer_r.convert_ids_to_tokens(a )
         lowercase__			    =						tokenizer_p.convert_ids_to_tokens(a )

         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(a    ,  a )
         self.assertListEqual(a    ,  a )

         lowercase__			    =						False
         lowercase__			    =						self.rust_tokenizer_class.from_pretrained(a    ,  **a )
         lowercase__			    =						self.tokenizer_class.from_pretrained(a    ,  **a )

         lowercase__			    =						tokenizer_r.encode(a    ,  add_special_tokens=a )
         lowercase__			    =						tokenizer_p.encode(a    ,  add_special_tokens=a )

         lowercase__			    =						tokenizer_r.convert_ids_to_tokens(a )
         lowercase__			    =						tokenizer_p.convert_ids_to_tokens(a )

         # it is expected that only the first Chinese character is not preceded by "##".
         lowercase__			    =						[
             f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a )
         ]
         self.assertListEqual(a    ,  a )
         self.assertListEqual(a    ,  a )









from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class        SCREAMING_SNAKE_CASE     (UpperCAmelCase   ):
   _UpperCamelCase					:	Tuple				      = 'ClapFeatureExtractor'
   _UpperCamelCase					:	Union[str, Any]				      = ('RobertaTokenizer', 'RobertaTokenizerFast')


   def __init__(					self    :       List[Any]    ,  a    :       int    ,  a    :       str )->     Any:





     """simple docstring"""







     super().__init__(a    ,  a )


   def __call__(					self    :       Any    ,  a    :       Tuple=None    ,  a    :       Optional[int]=None    ,  a    :       int=None    ,  **a    :       Optional[int] )->     Union[str, Any]:





     """simple docstring"""







     lowercase__			    =						kwargs.pop('sampling_rate'    ,  a )

     if text is None and audios is None:
       raise ValueError('You have to specify either text or audios. Both cannot be none.' )

     if text is not None:
       lowercase__			    =						self.tokenizer(a    ,  return_tensors=a    ,  **a )

     if audios is not None:
       lowercase__			    =						self.feature_extractor(
           a    ,  sampling_rate=a    ,  return_tensors=a    ,  **a )

     if text is not None and audios is not None:
       lowercase__			    =						audio_features.input_features
       return encoding
     elif text is not None:
       return encoding
     else:
       return BatchEncoding(data=dict(**a )    ,  tensor_type=a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       str    ,  *a    :       Dict    ,  **a    :       int )->     Optional[int]:





     """simple docstring"""







     return self.tokenizer.batch_decode(*a    ,  **a )


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[Any]    ,  *a    :       int    ,  **a    :       Dict )->     Dict:





     """simple docstring"""







     return self.tokenizer.decode(*a    ,  **a )


   @property
   def  SCREAMING_SNAKE_CASE_							(					self    :       Union[str, Any] )->     Optional[int]:





     """simple docstring"""







     lowercase__			    =						self.tokenizer.model_input_names
     lowercase__			    =						self.feature_extractor.model_input_names
     return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names ) )


