




import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY







@is_pipeline_test
class 		lowercase							(     unittest.TestCase	):
     _a											=		MODEL_FOR_MASKED_LM_MAPPING
     _a											=		TF_MODEL_FOR_MASKED_LM_MAPPING

     def       a__			(						self       )		->							Tuple:
           super().tearDown()
           # clean-up as much as possible GPU memory occupied by PyTorch
           gc.collect()
           if is_torch_available():
                 import torch

                 torch.cuda.empty_cache()

     @require_tf
     def       a__			(						self       )		->							Any:
           _A							:  Optional[Any]    =		pipeline(task="""fill-mask"""	,  model="""sshleifer/tiny-distilroberta-base"""	,  top_k=2	,  framework="""tf"""       )
           _A							:  Optional[int]    =		unmasker("""My name is <mask>"""       )
           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   {"""sequence""": """My name is grouped""", """score""": 2.1e-05, """token""": 3_8015, """token_str""": """ grouped"""},
                   {"""sequence""": """My name is accuser""", """score""": 2.1e-05, """token""": 2_5506, """token_str""": """ accuser"""},
               ]	,  )

           _A							:  Tuple    =		unmasker("""The largest city in France is <mask>"""       )
           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   {
                       """sequence""": """The largest city in France is grouped""",
                       """score""": 2.1e-05,
                       """token""": 3_8015,
                       """token_str""": """ grouped""",
                   },
                   {
                       """sequence""": """The largest city in France is accuser""",
                       """score""": 2.1e-05,
                       """token""": 2_5506,
                       """token_str""": """ accuser""",
                   },
               ]	,  )

           _A							:  List[str]    =		unmasker("""My name is <mask>"""	,  targets=[""" Patrick""", """ Clara""", """ Teven"""]	,  top_k=3       )
           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   {"""sequence""": """My name is Clara""", """score""": 2e-05, """token""": 1_3606, """token_str""": """ Clara"""},
                   {"""sequence""": """My name is Patrick""", """score""": 2e-05, """token""": 3499, """token_str""": """ Patrick"""},
                   {"""sequence""": """My name is Te""", """score""": 1.9e-05, """token""": 2941, """token_str""": """ Te"""},
               ]	,  )

     @require_torch
     def       a__			(						self       )		->							str:
           _A							:  Any    =		pipeline(task="""fill-mask"""	,  model="""sshleifer/tiny-distilroberta-base"""	,  top_k=2	,  framework="""pt"""       )

           _A							:  List[Any]    =		unmasker("""My name is <mask>"""       )
           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   {"""sequence""": """My name is Maul""", """score""": 2.2e-05, """token""": 3_5676, """token_str""": """ Maul"""},
                   {"""sequence""": """My name isELS""", """score""": 2.2e-05, """token""": 1_6416, """token_str""": """ELS"""},
               ]	,  )

           _A							:  Optional[Any]    =		unmasker("""The largest city in France is <mask>"""       )
           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   {
                       """sequence""": """The largest city in France is Maul""",
                       """score""": 2.2e-05,
                       """token""": 3_5676,
                       """token_str""": """ Maul""",
                   },
                   {"""sequence""": """The largest city in France isELS""", """score""": 2.2e-05, """token""": 1_6416, """token_str""": """ELS"""},
               ]	,  )

           _A							:  Optional[int]    =		unmasker("""My name is <mask>"""	,  targets=[""" Patrick""", """ Clara""", """ Teven"""]	,  top_k=3       )
           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   {"""sequence""": """My name is Patrick""", """score""": 2.1e-05, """token""": 3499, """token_str""": """ Patrick"""},
                   {"""sequence""": """My name is Te""", """score""": 2e-05, """token""": 2941, """token_str""": """ Te"""},
                   {"""sequence""": """My name is Clara""", """score""": 2e-05, """token""": 1_3606, """token_str""": """ Clara"""},
               ]	,  )

           _A							:  str    =		unmasker("""My name is <mask> <mask>"""	,  top_k=2       )

           self.assertEqual(
               nested_simplify(_a	,  decimals=6       )	,  [
                   [
                       {
                           """score""": 2.2e-05,
                           """token""": 3_5676,
                           """token_str""": """ Maul""",
                           """sequence""": """<s>My name is Maul<mask></s>""",
                       },
                       {"""score""": 2.2e-05, """token""": 1_6416, """token_str""": """ELS""", """sequence""": """<s>My name isELS<mask></s>"""},
                   ],
                   [
                       {
                           """score""": 2.2e-05,
                           """token""": 3_5676,
                           """token_str""": """ Maul""",
                           """sequence""": """<s>My name is<mask> Maul</s>""",
                       },
                       {"""score""": 2.2e-05, """token""": 1_6416, """token_str""": """ELS""", """sequence""": """<s>My name is<mask>ELS</s>"""},
                   ],
               ]	,  )

     @require_torch_gpu
     def       a__			(						self       )		->							Union[str, Any]:
           _A							:  int    =		pipeline("""fill-mask"""	,  model="""hf-internal-testing/tiny-random-distilbert"""	,  device=0	,  framework="""pt"""       )

           # convert model to fp16
           pipe.model.half()

           _A							:  Optional[Any]    =		pipe("""Paris is the [MASK] of France."""       )
           # We actually don't care about the result, we just want to make sure
           # it works, meaning the float16 tensor got casted back to float32
           # for postprocessing.
           self.assertIsInstance(_a	,  _a       )

     @slow
     @require_torch
     def       a__			(						self       )		->							Optional[int]:
           _A							:  Optional[Any]    =		pipeline(task="""fill-mask"""	,  model="""distilroberta-base"""	,  top_k=2	,  framework="""pt"""       )
           self.run_large_test(_a       )

     @slow
     @require_tf
     def       a__			(						self       )		->							Tuple:
           _A							:  str    =		pipeline(task="""fill-mask"""	,  model="""distilroberta-base"""	,  top_k=2	,  framework="""tf"""       )
           self.run_large_test(_a       )

     def       a__			(						self	,  _a       )		->							Tuple:
           _A							:  Optional[int]    =		unmasker("""My name is <mask>"""       )
           self.assertEqual(
               nested_simplify(_a       )	,  [
                   {"""sequence""": """My name is John""", """score""": 0.008, """token""": 610, """token_str""": """ John"""},
                   {"""sequence""": """My name is Chris""", """score""": 0.007, """token""": 1573, """token_str""": """ Chris"""},
               ]	,  )
           _A							:  int    =		unmasker("""The largest city in France is <mask>"""       )
           self.assertEqual(
               nested_simplify(_a       )	,  [
                   {
                       """sequence""": """The largest city in France is Paris""",
                       """score""": 0.251,
                       """token""": 2201,
                       """token_str""": """ Paris""",
                   },
                   {
                       """sequence""": """The largest city in France is Lyon""",
                       """score""": 0.214,
                       """token""": 1_2790,
                       """token_str""": """ Lyon""",
                   },
               ]	,  )

           _A							:  Optional[Any]    =		unmasker("""My name is <mask>"""	,  targets=[""" Patrick""", """ Clara""", """ Teven"""]	,  top_k=3       )
           self.assertEqual(
               nested_simplify(_a       )	,  [
                   {"""sequence""": """My name is Patrick""", """score""": 0.005, """token""": 3499, """token_str""": """ Patrick"""},
                   {"""sequence""": """My name is Clara""", """score""": 0.000, """token""": 1_3606, """token_str""": """ Clara"""},
                   {"""sequence""": """My name is Te""", """score""": 0.000, """token""": 2941, """token_str""": """ Te"""},
               ]	,  )

     @require_torch
     def       a__			(						self       )		->							Tuple:
           _A							:  List[str]    =		pipeline(task="""fill-mask"""	,  model="""sshleifer/tiny-distilroberta-base"""	,  framework="""pt"""       )
           _A							:  str    =		None
           _A							:  Union[str, Any]    =		None
           self.run_pipeline_test(_a	,  []       )

     @require_tf
     def       a__			(						self       )		->							Union[str, Any]:
           _A							:  Tuple    =		pipeline(task="""fill-mask"""	,  model="""sshleifer/tiny-distilroberta-base"""	,  framework="""tf"""       )
           _A							:  Any    =		None
           _A							:  Dict    =		None
           self.run_pipeline_test(_a	,  []       )

     def       a__			(						self	,  _a	,  _a	,  _a       )		->							Any:
           if tokenizer is None or tokenizer.mask_token_id is None:
                 self.skipTest("""The provided tokenizer has no mask token, (probably reformer or wav2vec2)"""       )

           _A							:  Optional[Any]    =		FillMaskPipeline(model=_a	,  tokenizer=_a       )
           _A							:  Tuple    =		[
               F'''This is another {tokenizer.mask_token} test''',
           ]
           return fill_masker, examples

     def       a__			(						self	,  _a	,  _a       )		->							Dict:
           _A							:  Dict    =		fill_masker.tokenizer
           _A							:  List[str]    =		fill_masker.model

           _A							:  List[str]    =		fill_masker(
               F'''This is a {tokenizer.mask_token}'''	,  )
           self.assertEqual(
               _a	,  [
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
               ]	,  )

           _A							:  Optional[Any]    =		fill_masker([F'''This is a {tokenizer.mask_token}''']       )
           self.assertEqual(
               _a	,  [
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
               ]	,  )

           _A							:  List[str]    =		fill_masker([F'''This is a {tokenizer.mask_token}''', F'''Another {tokenizer.mask_token} great test.''']       )
           self.assertEqual(
               _a	,  [
                   [
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   ],
                   [
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   ],
               ]	,  )

           with self.assertRaises(_a       ):
                 fill_masker([None]       )
           # No mask_token is not supported
           with self.assertRaises(_a       ):
                 fill_masker("""This is"""       )

           self.run_test_top_k(_a	,  _a       )
           self.run_test_targets(_a	,  _a       )
           self.run_test_top_k_targets(_a	,  _a       )
           self.fill_mask_with_duplicate_targets_and_top_k(_a	,  _a       )
           self.fill_mask_with_multiple_masks(_a	,  _a       )

     def       a__			(						self	,  _a	,  _a       )		->							List[str]:
           _A							:  int    =		tokenizer.get_vocab()
           _A							:  str    =		sorted(vocab.keys()       )[:2]
           # Pipeline argument
           _A							:  Tuple    =		FillMaskPipeline(model=_a	,  tokenizer=_a	,  targets=_a       )
           _A							:  Optional[int]    =		fill_masker(F'''This is a {tokenizer.mask_token}'''       )
           self.assertEqual(
               _a	,  [
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
               ]	,  )
           _A							:  List[str]    =		{vocab[el] for el in targets}
           self.assertEqual({el["""token"""] for el in outputs}	,  _a       )
           _A							:  Union[str, Any]    =		[tokenizer.decode([x]       ) for x in target_ids]
           self.assertEqual({el["""token_str"""] for el in outputs}	,  set(_a       )       )

           # Call argument
           _A							:  str    =		FillMaskPipeline(model=_a	,  tokenizer=_a       )
           _A							:  Optional[int]    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  targets=_a       )
           self.assertEqual(
               _a	,  [
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
               ]	,  )
           _A							:  int    =		{vocab[el] for el in targets}
           self.assertEqual({el["""token"""] for el in outputs}	,  _a       )
           _A							:  Any    =		[tokenizer.decode([x]       ) for x in target_ids]
           self.assertEqual({el["""token_str"""] for el in outputs}	,  set(_a       )       )

           # Score equivalence
           _A							:  int    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  targets=_a       )
           _A							:  Optional[int]    =		[top_mask["""token_str"""] for top_mask in outputs]
           _A							:  Union[str, Any]    =		[top_mask["""score"""] for top_mask in outputs]

           # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
           if set(_a       ) == set(_a       ):
                 _A							:  Tuple    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  targets=_a       )
                 _A							:  Union[str, Any]    =		[top_mask["""score"""] for top_mask in unmasked_targets]
                 self.assertEqual(nested_simplify(_a       )	,  nested_simplify(_a       )       )

           # Raises with invalid
           with self.assertRaises(_a       ):
                 _A							:  str    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  targets=[]       )
           # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
           if "" not in tokenizer.get_vocab():
                 with self.assertRaises(_a       ):
                       _A							:  Optional[Any]    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  targets=[""""""]       )
                 with self.assertRaises(_a       ):
                       _A							:  int    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  targets=""""""       )

     def       a__			(						self	,  _a	,  _a       )		->							Optional[Any]:
           _A							:  str    =		FillMaskPipeline(model=_a	,  tokenizer=_a	,  top_k=2       )
           _A							:  str    =		fill_masker(F'''This is a {tokenizer.mask_token}'''       )
           self.assertEqual(
               _a	,  [
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
               ]	,  )

           _A							:  Union[str, Any]    =		FillMaskPipeline(model=_a	,  tokenizer=_a       )
           _A							:  Union[str, Any]    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  top_k=2       )
           self.assertEqual(
               _a	,  [
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
               ]	,  )
           self.assertEqual(nested_simplify(_a       )	,  nested_simplify(_a       )       )

     def       a__			(						self	,  _a	,  _a       )		->							List[Any]:
           _A							:  Union[str, Any]    =		tokenizer.get_vocab()
           _A							:  int    =		FillMaskPipeline(model=_a	,  tokenizer=_a       )

           # top_k=2, ntargets=3
           _A							:  List[str]    =		sorted(vocab.keys()       )[:3]
           _A							:  Tuple    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  top_k=2	,  targets=_a       )

           # If we use the most probably targets, and filter differently, we should still
           # have the same results
           _A							:  Any    =		[el["""token_str"""] for el in sorted(_a	,  key=lambda _a       : x["score"]	,  reverse=_a       )]
           # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
           if set(_a       ).issubset(_a       ):
                 _A							:  Any    =		fill_masker(F'''This is a {tokenizer.mask_token}'''	,  top_k=3	,  targets=_a       )
                 # They should yield exactly the same result
                 self.assertEqual(nested_simplify(_a       )	,  nested_simplify(_a       )       )

     def       a__			(						self	,  _a	,  _a       )		->							str:
           _A							:  Optional[int]    =		FillMaskPipeline(model=_a	,  tokenizer=_a       )
           _A							:  List[Any]    =		tokenizer.get_vocab()
           # String duplicates + id duplicates
           _A							:  Optional[Any]    =		sorted(vocab.keys()       )[:3]
           _A							:  Optional[Any]    =		[targets[0], targets[1], targets[0], targets[2], targets[1]]
           _A							:  Union[str, Any]    =		fill_masker(F'''My name is {tokenizer.mask_token}'''	,  targets=_a	,  top_k=10       )

           # The target list contains duplicates, so we can't output more
           # than them
           self.assertEqual(len(_a       )	,  3       )



     def       a__			(						self	,  _a	,  _a       )		->							Tuple:
           _A							:  Any    =		FillMaskPipeline(model=_a	,  tokenizer=_a       )

           _A							:  Optional[Any]    =		fill_masker(
               F'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''	,  top_k=2       )
           self.assertEqual(
               _a	,  [
                   [
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   ],
                   [
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   ],
                   [
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                       {"""sequence""": ANY(_a       ), """score""": ANY(_a       ), """token""": ANY(_a       ), """token_str""": ANY(_a       )},
                   ],
               ]	,  )







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		lowercase							(     UpperCamelCase__,unittest.TestCase	):
     _a											=		CLIPTokenizer
     _a											=		CLIPTokenizerFast
     _a											=		True
     _a											=		{}
     _a											=		False

     def       a__			(						self       )		->							Optional[Any]:
           super().setUp()

           # fmt: off
           _A							:  int    =		["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
           # fmt: on
           _A							:  str    =		dict(zip(_a	,  range(len(_a       )       )       )       )
           _A							:  Optional[int]    =		["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
           _A							:  str    =		{"""unk_token""": """<unk>"""}

           _A							:  Union[str, Any]    =		os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["""vocab_file"""]       )
           _A							:  Union[str, Any]    =		os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["""merges_file"""]       )
           with open(self.vocab_file	,  """w"""	,  encoding="""utf-8"""       ) as fp:
                 fp.write(json.dumps(_a       ) + """\n"""       )
           with open(self.merges_file	,  """w"""	,  encoding="""utf-8"""       ) as fp:
                 fp.write("""\n""".join(_a       )       )

     def       a__			(						self	,  **_a       )		->							List[Any]:
           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizer.from_pretrained(self.tmpdirname	,  **_a       )

     def       a__			(						self	,  **_a       )		->							Union[str, Any]:
           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname	,  **_a       )

     def       a__			(						self	,  _a       )		->							str:
           _A							:  Tuple    =		"""lower newer"""
           _A							:  Optional[Any]    =		"""lower newer"""
           return input_text, output_text

     def       a__			(						self       )		->							List[Any]:
           _A							:  Optional[int]    =		CLIPTokenizer(self.vocab_file	,  self.merges_file	,  **self.special_tokens_map       )
           _A							:  str    =		"""lower newer"""
           _A							:  List[str]    =		["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
           _A							:  int    =		tokenizer.tokenize(_a       )
           self.assertListEqual(_a	,  _a       )

           _A							:  str    =		tokens + [tokenizer.unk_token]
           _A							:  Dict    =		[10, 2, 16, 9, 3, 2, 16, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(_a       )	,  _a       )

     @require_ftfy
     def       a__			(						self       )		->							Tuple:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                       _A							:  Dict    =		self.tokenizer_class.from_pretrained(_a	,  **_a       )
                       _A							:  str    =		self.rust_tokenizer_class.from_pretrained(_a	,  **_a       )

                       _A							:  List[str]    =		"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
                       _A							:  Dict    =		tokenizer_s.tokenize(_a       )
                       _A							:  Dict    =		tokenizer_r.tokenize(_a       )

                       self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                       # with Tilde) encoded in 2 different ways
                       _A							:  Tuple    =		"""xa\u0303y""" + """ """ + """x\xe3y"""
                       _A							:  Dict    =		tokenizer_s.tokenize(_a       )
                       _A							:  Dict    =		tokenizer_r.tokenize(_a       )

                       self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on unicode of space type
                       _A							:  Any    =		[
                           """\u0009""",  # (horizontal tab, '\t')
                           """\u000B""",  # (vertical tab)
                           """\u000C""",  # (form feed)
                           """\u0020""",  # (space, ' ')
                           """\u200E""",  # (left-to-right mark):w
                           """\u200F""",  # (right-to-left mark)
                       ]
                       for unicode_seq in spaces_unicodes:
                             _A							:  Optional[int]    =		tokenizer_s.tokenize(_a       )
                             _A							:  List[Any]    =		tokenizer_r.tokenize(_a       )

                             self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on unicode of line break type
                       _A							:  int    =		[
                           """\u000A""",  # (line feed, '\n')
                           """\r\n""",  # (carriage return and line feed, '\r\n')
                           """\u000D""",  # (carriage return, '\r')
                           """\r""",  # (carriage return, '\r')
                           """\u000D""",  # (carriage return, '\r')
                           """\u2028""",  # (line separator)
                           """\u2029""",  # (paragraph separator)
                           # "\u0085", # (next line)
                       ]

                       # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                       # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                       # space (and thus into an empty list).

                       for unicode_seq in line_break_unicodes:
                             _A							:  Optional[Any]    =		tokenizer_s.tokenize(_a       )
                             _A							:  Tuple    =		tokenizer_r.tokenize(_a       )

                             self.assertListEqual(_a	,  _a       )

     def       a__			(						self       )		->							Any:
           # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                       _A							:  List[Any]    =		"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                       _A							:  str    =		F'''{text_of_1_token} {text_of_1_token}'''

                       _A							:  str    =		self.rust_tokenizer_class.from_pretrained(
                           _a	,  use_fast=_a	,  )
                       _A							:  Dict    =		tokenizer_r(_a	,  return_offsets_mapping=_a	,  add_special_tokens=_a       )
                       self.assertEqual(encoding.offset_mapping[0]	,  (0, len(_a       ))       )
                       self.assertEqual(
                           encoding.offset_mapping[1]	,  (len(_a       ) + 1, len(_a       ) + 1 + len(_a       ))	,  )

                       _A							:  Any    =		F''' {text}'''

                       _A							:  List[Any]    =		self.rust_tokenizer_class.from_pretrained(
                           _a	,  use_fast=_a	,  )
                       _A							:  Optional[int]    =		tokenizer_r(_a	,  return_offsets_mapping=_a	,  add_special_tokens=_a       )
                       self.assertEqual(encoding.offset_mapping[0]	,  (1, 1 + len(_a       ))       )
                       self.assertEqual(
                           encoding.offset_mapping[1]	,  (1 + len(_a       ) + 1, 1 + len(_a       ) + 1 + len(_a       ))	,  )

     def       a__			(						self       )		->							List[Any]:
           # Test related to the breaking change introduced in transformers v4.17.0
           # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
           with self.assertRaises(_a       ) as context:
                 self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""       )

           self.assertTrue(
               context.exception.args[0].startswith(
                   """The `backend_tokenizer` provided does not match the expected format."""       )       )

     @require_ftfy
     def       a__			(						self       )		->							str:
           super().test_tokenization_python_rust_equals()



     def       a__			(						self       )		->							Union[str, Any]:
           # CLIP always lower cases letters
           pass

