



"""simple docstring"""

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class  _UpperCAmelCase (    UpperCAmelCase__					,  unittest.TestCase):
 _lowerCAmelCase     :    Dict	      =	BertTokenizer
 _lowerCAmelCase     :    Optional[int]	      =	BertTokenizerFast
 _lowerCAmelCase     :    Optional[Any]	      =	True
 _lowerCAmelCase     :    int	      =	True
 _lowerCAmelCase     :    Any	      =	filter_non_english

 def 		_snake_case      (  self	: Optional[Any]       ):
     super().setUp()

     snake_case_				:  List[str]       =			[
         """[UNK]""",
         """[CLS]""",
         """[SEP]""",
         """[PAD]""",
         """[MASK]""",
         """want""",
         """##want""",
         """##ed""",
         """wa""",
         """un""",
         """runn""",
         """##ing""",
         """,""",
         """low""",
         """lowest""",
     ]
     snake_case_				:  Any       =			os.path.join(self.tmpdirname						,  VOCAB_FILES_NAMES['''vocab_file''']       )
     with open(self.vocab_file						,  '''w'''						,  encoding='''utf-8'''       ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )

 def 		_snake_case      (  self	: Optional[int]						,  lowercase_	: Any       ):
     snake_case_				:  Union[str, Any]       =			"""UNwant\u00E9d,running"""
     snake_case_				:  Optional[Any]       =			"""unwanted, running"""
     return input_text, output_text

 def 		_snake_case      (  self	: Any       ):
     snake_case_				:  List[str]       =			self.tokenizer_class(self.vocab_file       )

     snake_case_				:  Dict       =			tokenizer.tokenize('''UNwant\u00E9d,running'''       )
     self.assertListEqual(lowercase_						,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_       )						,  [9, 6, 7, 12, 10, 11]       )

 def 		_snake_case      (  self	: Optional[int]       ):




     if not self.test_rust_tokenizer:
         return

     snake_case_				:  Optional[Any]       =			self.get_tokenizer()
     snake_case_				:  Any       =			self.get_rust_tokenizer()

     snake_case_				:  str       =			"""UNwant\u00E9d,running"""

     snake_case_				:  List[str]       =			tokenizer.tokenize(lowercase_       )
     snake_case_				:  List[Any]       =			rust_tokenizer.tokenize(lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     snake_case_				:  Dict       =			tokenizer.encode(lowercase_						,  add_special_tokens=lowercase_       )
     snake_case_				:  Optional[Any]       =			rust_tokenizer.encode(lowercase_						,  add_special_tokens=lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     snake_case_				:  int       =			self.get_rust_tokenizer()
     snake_case_				:  Union[str, Any]       =			tokenizer.encode(lowercase_       )
     snake_case_				:  Optional[int]       =			rust_tokenizer.encode(lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     # With lower casing
     snake_case_				:  Optional[int]       =			self.get_tokenizer(do_lower_case=lowercase_       )
     snake_case_				:  List[str]       =			self.get_rust_tokenizer(do_lower_case=lowercase_       )

     snake_case_				:  str       =			"""UNwant\u00E9d,running"""

     snake_case_				:  Any       =			tokenizer.tokenize(lowercase_       )
     snake_case_				:  str       =			rust_tokenizer.tokenize(lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     snake_case_				:  Union[str, Any]       =			tokenizer.encode(lowercase_						,  add_special_tokens=lowercase_       )
     snake_case_				:  Optional[int]       =			rust_tokenizer.encode(lowercase_						,  add_special_tokens=lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     snake_case_				:  Optional[int]       =			self.get_rust_tokenizer()
     snake_case_				:  Union[str, Any]       =			tokenizer.encode(lowercase_       )
     snake_case_				:  Any       =			rust_tokenizer.encode(lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

 def 		_snake_case      (  self	: List[Any]       ):
     snake_case_				:  Union[str, Any]       =			BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       )						,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )

 def 		_snake_case      (  self	: Any       ):
     snake_case_				:  Dict       =			BasicTokenizer(do_lower_case=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )						,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,  ['''hello''']       )

 def 		_snake_case      (  self	: List[str]       ):
     snake_case_				:  List[str]       =			BasicTokenizer(do_lower_case=lowercase_						,  strip_accents=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,  ['''h\u00E9llo''']       )

 def 		_snake_case      (  self	: List[Any]       ):
     snake_case_				:  Tuple       =			BasicTokenizer(do_lower_case=lowercase_						,  strip_accents=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,  ['''hello''']       )

 def 		_snake_case      (  self	: Dict       ):
     snake_case_				:  int       =			BasicTokenizer(do_lower_case=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,  ['''hello''']       )

 def 		_snake_case      (  self	: List[Any]       ):
     snake_case_				:  Any       =			BasicTokenizer(do_lower_case=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )						,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )

 def 		_snake_case      (  self	: Union[str, Any]       ):
     snake_case_				:  List[str]       =			BasicTokenizer(do_lower_case=lowercase_						,  strip_accents=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )

 def 		_snake_case      (  self	: str       ):
     snake_case_				:  List[Any]       =			BasicTokenizer(do_lower_case=lowercase_						,  strip_accents=lowercase_       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )

 def 		_snake_case      (  self	: Union[str, Any]       ):
     snake_case_				:  int       =			BasicTokenizer(do_lower_case=lowercase_						,  never_split=['''[UNK]''']       )

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       )						,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )

 def 		_snake_case      (  self	: Dict       ):
     snake_case_				:  int       =			BasicTokenizer()
     snake_case_				:  Tuple       =			"""a\n'll !!to?'d of, can't."""
     snake_case_				:  List[Any]       =			["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
     self.assertListEqual(tokenizer.tokenize(lowercase_       )						,  lowercase_       )

 def 		_snake_case      (  self	: Optional[int]       ):
     snake_case_				:  str       =			["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

     snake_case_				:  Union[str, Any]       =			{}
     for i, token in enumerate(lowercase_       ):
         snake_case_				:  Union[str, Any]       =			i
     snake_case_				:  Union[str, Any]       =			WordpieceTokenizer(vocab=lowercase_						,  unk_token='''[UNK]'''       )

     self.assertListEqual(tokenizer.tokenize(''''''       )						,  []       )

     self.assertListEqual(tokenizer.tokenize('''unwanted running'''       )						,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

     self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       )						,  ['''[UNK]''', '''runn''', '''##ing''']       )

 def 		_snake_case      (  self	: Optional[int]       ):
     self.assertTrue(_is_whitespace(''' '''       )       )
     self.assertTrue(_is_whitespace('''\t'''       )       )
     self.assertTrue(_is_whitespace('''\r'''       )       )
     self.assertTrue(_is_whitespace('''\n'''       )       )
     self.assertTrue(_is_whitespace('''\u00A0'''       )       )

     self.assertFalse(_is_whitespace('''A'''       )       )
     self.assertFalse(_is_whitespace('''-'''       )       )

 def 		_snake_case      (  self	: Any       ):
     self.assertTrue(_is_control('''\u0005'''       )       )

     self.assertFalse(_is_control('''A'''       )       )
     self.assertFalse(_is_control(''' '''       )       )
     self.assertFalse(_is_control('''\t'''       )       )
     self.assertFalse(_is_control('''\r'''       )       )

 def 		_snake_case      (  self	: List[Any]       ):
     self.assertTrue(_is_punctuation('''-'''       )       )
     self.assertTrue(_is_punctuation('''$'''       )       )
     self.assertTrue(_is_punctuation('''`'''       )       )
     self.assertTrue(_is_punctuation('''.'''       )       )

     self.assertFalse(_is_punctuation('''A'''       )       )
     self.assertFalse(_is_punctuation(''' '''       )       )

 def 		_snake_case      (  self	: List[Any]       ):
     snake_case_				:  Tuple       =			self.get_tokenizer()
     snake_case_				:  Optional[int]       =			self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(lowercase_       ) for t in ['''Test''', '''\xad''', '''test''']]						,  [['''[UNK]'''], [], ['''[UNK]''']]       )

     self.assertListEqual(
         [rust_tokenizer.tokenize(lowercase_       ) for t in ['''Test''', '''\xad''', '''test''']]						,  [['''[UNK]'''], [], ['''[UNK]''']]       )

 @slow
 def 		_snake_case      (  self	: List[str]       ):
     snake_case_				:  Optional[Any]       =			self.tokenizer_class.from_pretrained('''bert-base-uncased'''       )

     snake_case_				:  int       =			tokenizer.encode('''sequence builders'''						,  add_special_tokens=lowercase_       )
     snake_case_				:  Tuple       =			tokenizer.encode('''multi-sequence build'''						,  add_special_tokens=lowercase_       )

     snake_case_				:  Union[str, Any]       =			tokenizer.build_inputs_with_special_tokens(lowercase_       )
     snake_case_				:  int       =			tokenizer.build_inputs_with_special_tokens(lowercase_						,  lowercase_       )

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]

 def 		_snake_case      (  self	: Tuple       ):




     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
             snake_case_				:  List[Any]       =			self.rust_tokenizer_class.from_pretrained(lowercase_						,  **lowercase_       )

             snake_case_				:  Dict       =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
             snake_case_				:  Optional[Any]       =			tokenizer_r.encode_plus(
                 lowercase_						,  return_attention_mask=lowercase_						,  return_token_type_ids=lowercase_						,  return_offsets_mapping=lowercase_						,  add_special_tokens=lowercase_						,  )

             snake_case_				:  Any       =			tokenizer_r.do_lower_case if hasattr(lowercase_						,  '''do_lower_case'''       ) else False
             snake_case_				:  int       =			(
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), """A"""),
                     ((1, 2), ""","""),
                     ((3, 5), """na"""),
                     ((5, 6), """##ï"""),
                     ((6, 8), """##ve"""),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), """Allen"""),
                     ((21, 23), """##NL"""),
                     ((23, 24), """##P"""),
                     ((25, 33), """sentence"""),
                     ((33, 34), """."""),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), """a"""),
                     ((1, 2), ""","""),
                     ((3, 8), """naive"""),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), """allen"""),
                     ((21, 23), """##nl"""),
                     ((23, 24), """##p"""),
                     ((25, 33), """sentence"""),
                     ((33, 34), """."""),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]						,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
             self.assertEqual([e[0] for e in expected_results]						,  tokens['''offset_mapping''']       )






 def 		_snake_case      (  self	: List[str]       ):
     snake_case_				:  Dict       =			["""的""", """人""", """有"""]
     snake_case_				:  List[str]       =			"""""".join(lowercase_       )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
             snake_case_				:  Tuple       =			True
             snake_case_				:  str       =			self.tokenizer_class.from_pretrained(lowercase_						,  **lowercase_       )
             snake_case_				:  List[str]       =			self.rust_tokenizer_class.from_pretrained(lowercase_						,  **lowercase_       )

             snake_case_				:  Optional[int]       =			tokenizer_p.encode(lowercase_						,  add_special_tokens=lowercase_       )
             snake_case_				:  Optional[int]       =			tokenizer_r.encode(lowercase_						,  add_special_tokens=lowercase_       )

             snake_case_				:  List[Any]       =			tokenizer_r.convert_ids_to_tokens(lowercase_       )
             snake_case_				:  List[str]       =			tokenizer_p.convert_ids_to_tokens(lowercase_       )

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(lowercase_						,  lowercase_       )
             self.assertListEqual(lowercase_						,  lowercase_       )

             snake_case_				:  Optional[int]       =			False
             snake_case_				:  List[str]       =			self.rust_tokenizer_class.from_pretrained(lowercase_						,  **lowercase_       )
             snake_case_				:  Tuple       =			self.tokenizer_class.from_pretrained(lowercase_						,  **lowercase_       )

             snake_case_				:  Dict       =			tokenizer_r.encode(lowercase_						,  add_special_tokens=lowercase_       )
             snake_case_				:  Any       =			tokenizer_p.encode(lowercase_						,  add_special_tokens=lowercase_       )

             snake_case_				:  str       =			tokenizer_r.convert_ids_to_tokens(lowercase_       )
             snake_case_				:  Union[str, Any]       =			tokenizer_p.convert_ids_to_tokens(lowercase_       )

             # it is expected that only the first Chinese character is not preceded by "##".
             snake_case_				:  int       =			[
                 f"##{token}" if idx != 0 else token for idx, token in enumerate(lowercase_       )
             ]
             self.assertListEqual(lowercase_						,  lowercase_       )
             self.assertListEqual(lowercase_						,  lowercase_       )










"""simple docstring"""

import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
     import torch

     from transformers import AutoModelForImageClassification

if is_vision_available():
     from transformers import AutoImageProcessor




@require_torch
@require_vision
class  _UpperCAmelCase (    unittest.TestCase):

 @slow
 def 		_snake_case      (  self	: List[Any]       ):
     snake_case_				:  List[Any]       =			AutoImageProcessor.from_pretrained('''microsoft/dit-base-finetuned-rvlcdip'''       )
     snake_case_				:  List[Any]       =			AutoModelForImageClassification.from_pretrained('''microsoft/dit-base-finetuned-rvlcdip'''       )
     model.to(lowercase_       )

     from datasets import load_dataset

     snake_case_				:  str       =			load_dataset('''nielsr/rvlcdip-demo'''       )

     snake_case_				:  Tuple       =			dataset['''train'''][0]['''image'''].convert('''RGB'''       )

     snake_case_				:  Union[str, Any]       =			image_processor(lowercase_						,  return_tensors='''pt'''       ).to(lowercase_       )

     # forward pass
     with torch.no_grad():
         snake_case_				:  List[Any]       =			model(**lowercase_       )
         snake_case_				:  int       =			outputs.logits

     snake_case_				:  Any       =			torch.Size((1, 16)       )
     self.assertEqual(logits.shape						,  lowercase_       )

     snake_case_				:  Any       =			torch.tensor(
         [-0.41_58, -0.40_92, -0.43_47]						,  device=lowercase_						,  dtype=torch.float						,  )
     self.assertTrue(torch.allclose(logits[0, :3]						,  lowercase_						,  atol=1E-4       )       )





