
'''simple docstring'''



import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     int:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				int(SCREAMING_SNAKE_CASE_						)
   _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				t // 36_00, (t // 60) % 60, t % 60
   return F"{h}:{m:02d}:{s:02d}" if h != 0 else F"{m:02d}:{s:02d}"
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_=3_00						)     ->     Union[str, Any]:




   """simple docstring"""
   return F"\n    <div>\n      {prefix}\n      <progress value='{value}' max='{total}' style='width:{width}px; height:20px; vertical-align: middle;'></progress>\n      {label}\n    </div>\n    "
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				"""<table border=\"1\" class=\"dataframe\">\n"""
   html_code += """  <thead>\n <tr style="text-align: left;">\n"""
   for i in items[0]:
      html_code += F"      <th>{i}</th>\n"
   html_code += "    </tr>\n  </thead>\n  <tbody>\n"
   for line in items[1:]:
      html_code += "    <tr>\n"
      for elt in line:
         _SCREAMING_SNAKE_CASE      =				F"{elt:.6f}" if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						) else str(SCREAMING_SNAKE_CASE_						)
         html_code += F"      <td>{elt}</td>\n"
      html_code += "    </tr>\n"
   html_code += "  </tbody>\n</table><p>"
   return html_code

class    _a				:





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  5
       SCREAMING_SNAKE_CASE      =  0.2






       def __init__(							self       ,							A__       ,							A__ = None       ,							A__ = True       ,							A__ = None       ,							A__ = 3_00       ,							)    ->			int:
          _SCREAMING_SNAKE_CASE      =				total
          _SCREAMING_SNAKE_CASE      =				"""""" if prefix is None else prefix
          _SCREAMING_SNAKE_CASE      =				leave
          _SCREAMING_SNAKE_CASE      =				parent
          _SCREAMING_SNAKE_CASE      =				width
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				None






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = False       ,							A__ = None      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				value
          if comment is not None:
             _SCREAMING_SNAKE_CASE      =				comment
          if self.last_value is None:
             _SCREAMING_SNAKE_CASE      =				_SCREAMING_SNAKE_CASE      =				time.time()
             _SCREAMING_SNAKE_CASE      =				_SCREAMING_SNAKE_CASE      =				value
             _SCREAMING_SNAKE_CASE      =				_SCREAMING_SNAKE_CASE      =				None
             _SCREAMING_SNAKE_CASE      =				self.warmup
             _SCREAMING_SNAKE_CASE      =				1
             self.update_bar(A__      )
          elif value <= self.last_value and not force_update:
             return
          elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for       ,							self.total      ):
             if self.first_calls > 0:
                self.first_calls -= 1
             _SCREAMING_SNAKE_CASE      =				time.time()
             _SCREAMING_SNAKE_CASE      =				current_time - self.start_time
             # We could have value = self.start_value if the update is called twixe with the same start value.
             if value > self.start_value:
                _SCREAMING_SNAKE_CASE      =				self.elapsed_time / (value - self.start_value)
             else:
                _SCREAMING_SNAKE_CASE      =				None
             if value >= self.total:
                _SCREAMING_SNAKE_CASE      =				self.total
                _SCREAMING_SNAKE_CASE      =				None
                if not self.leave:
                   self.close()
             elif self.average_time_per_item is not None:
                _SCREAMING_SNAKE_CASE      =				self.average_time_per_item * (self.total - value)
             self.update_bar(A__      )
             _SCREAMING_SNAKE_CASE      =				value
             _SCREAMING_SNAKE_CASE      =				current_time
             if self.average_time_per_item is None:
                _SCREAMING_SNAKE_CASE      =				1
             else:
                _SCREAMING_SNAKE_CASE      =				max(int(self.update_every / self.average_time_per_item      )       ,							1      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__=None      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				""" """ * (len(str(self.total      )      ) - len(str(A__      )      )) + str(A__      )
          if self.elapsed_time is None:
             _SCREAMING_SNAKE_CASE      =				F"[{spaced_value}/{self.total} : < :"
          elif self.predicted_remaining is None:
             _SCREAMING_SNAKE_CASE      =				F"[{spaced_value}/{self.total} {format_time(self.elapsed_time      )}"
          else:
             _SCREAMING_SNAKE_CASE      =				(
                 F"[{spaced_value}/{self.total} {format_time(self.elapsed_time      )} <"
                 F" {format_time(self.predicted_remaining      )}"
             )
             self.label += F", {1/self.average_time_per_item:.2f} it/s"
          self.label += "]" if self.comment is None or len(self.comment      ) == 0 else F", {self.comment}]"
          self.display()






       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				html_progress_bar(self.value       ,							self.total       ,							self.prefix       ,							self.label       ,							self.width      )
          if self.parent is not None:
             # If this is a child bar, the parent will take care of the display.
             self.parent.display()
             return
          if self.output is None:
             _SCREAMING_SNAKE_CASE      =				disp.display(disp.HTML(self.html_code      )       ,							display_id=A__      )
          else:
             self.output.update(disp.HTML(self.html_code      )      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          if self.parent is None and self.output is not None:
             self.output.update(disp.HTML(""""""      )      )






class    _a				(_lowerCamelCase):





       """simple docstring"""




       def __init__(							self       ,							A__       ,							A__=None      )    ->			Optional[int]:
          super().__init__(A__      )
          _SCREAMING_SNAKE_CASE      =				None if column_names is None else [column_names]
          _SCREAMING_SNAKE_CASE      =				None






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				html_progress_bar(self.value       ,							self.total       ,							self.prefix       ,							self.label       ,							self.width      )
          if self.inner_table is not None:
             self.html_code += text_to_html_table(self.inner_table      )
          if self.child_bar is not None:
             self.html_code += self.child_bar.html_code
          if self.output is None:
             _SCREAMING_SNAKE_CASE      =				disp.display(disp.HTML(self.html_code      )       ,							display_id=A__      )
          else:
             self.output.update(disp.HTML(self.html_code      )      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			Any:
          if self.inner_table is None:
             _SCREAMING_SNAKE_CASE      =				[list(values.keys()      ), list(values.values()      )]
          else:
             _SCREAMING_SNAKE_CASE      =				self.inner_table[0]
             if len(self.inner_table      ) == 1:
                # We give a chance to update the column names at the first iteration
                for key in values.keys():
                   if key not in columns:
                      columns.append(A__      )
                _SCREAMING_SNAKE_CASE      =				columns
             self.inner_table.append([values[c] for c in columns]      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__=None       ,							A__=3_00      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				NotebookProgressBar(A__       ,							prefix=A__       ,							parent=self       ,							width=A__      )
          return self.child_bar






       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				None
          self.display()






class    _a				(_lowerCamelCase):





       """simple docstring"""




       def __init__(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				None
          _SCREAMING_SNAKE_CASE      =				False






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							**A__      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				"""Epoch""" if args.evaluation_strategy == IntervalStrategy.EPOCH else """Step"""
          _SCREAMING_SNAKE_CASE      =				0
          _SCREAMING_SNAKE_CASE      =				0
          _SCREAMING_SNAKE_CASE      =				[self.first_column] + ["""Training Loss"""]
          if args.evaluation_strategy != IntervalStrategy.NO:
             column_names.append("""Validation Loss"""      )
          _SCREAMING_SNAKE_CASE      =				NotebookTrainingTracker(state.max_steps       ,							A__      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							**A__      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				int(state.epoch      ) if int(state.epoch      ) == state.epoch else F"{state.epoch:.2f}"
          self.training_tracker.update(
              state.global_step + 1       ,							comment=F"Epoch {epoch}/{state.num_train_epochs}"       ,							force_update=self._force_next_update       ,							)
          _SCREAMING_SNAKE_CASE      =				False






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__=None       ,							**A__      )    ->			Union[str, Any]:
          if not has_length(A__      ):
             return
          if self.prediction_bar is None:
             if self.training_tracker is not None:
                _SCREAMING_SNAKE_CASE      =				self.training_tracker.add_child(len(A__      )      )
             else:
                _SCREAMING_SNAKE_CASE      =				NotebookProgressBar(len(A__      )      )
             self.prediction_bar.update(1      )
          else:
             self.prediction_bar.update(self.prediction_bar.value + 1      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							**A__      )    ->			List[Any]:
          if self.prediction_bar is not None:
             self.prediction_bar.close()
          _SCREAMING_SNAKE_CASE      =				None






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__=None       ,							**A__      )    ->			Optional[int]:
          # Only for when there is no evaluation
          if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
             _SCREAMING_SNAKE_CASE      =				{"""Training Loss""": logs["""loss"""]}
             # First column is necessarily Step sine we're not in epoch eval strategy
             _SCREAMING_SNAKE_CASE      =				state.global_step
             self.training_tracker.write_line(A__      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__=None       ,							**A__      )    ->			Dict:
          if self.training_tracker is not None:
             _SCREAMING_SNAKE_CASE      =				{"""Training Loss""": """No log""", """Validation Loss""": """No log"""}
             for log in reversed(state.log_history      ):
                if "loss" in log:
                   _SCREAMING_SNAKE_CASE      =				log["""loss"""]
                   break

             if self.first_column == "Epoch":
                _SCREAMING_SNAKE_CASE      =				int(state.epoch      )
             else:
                _SCREAMING_SNAKE_CASE      =				state.global_step
             _SCREAMING_SNAKE_CASE      =				"""eval"""
             for k in metrics:
                if k.endswith("""_loss"""      ):
                   _SCREAMING_SNAKE_CASE      =				re.sub(R"""\_loss$"""       ,							""""""       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				metrics.pop("""total_flos"""       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				metrics.pop("""epoch"""       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				metrics.pop(F"{metric_key_prefix}_runtime"       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				metrics.pop(F"{metric_key_prefix}_samples_per_second"       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				metrics.pop(F"{metric_key_prefix}_steps_per_second"       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				metrics.pop(F"{metric_key_prefix}_jit_compilation_time"       ,							A__      )
             for k, v in metrics.items():
                if k == F"{metric_key_prefix}_loss":
                   _SCREAMING_SNAKE_CASE      =				v
                else:
                   _SCREAMING_SNAKE_CASE      =				k.split("""_"""      )
                   _SCREAMING_SNAKE_CASE      =				""" """.join([part.capitalize() for part in splits[1:]]      )
                   _SCREAMING_SNAKE_CASE      =				v
             self.training_tracker.write_line(A__      )
             self.training_tracker.remove_child()
             _SCREAMING_SNAKE_CASE      =				None
             # Evaluation takes a long time so we should force the next update.
             _SCREAMING_SNAKE_CASE      =				True






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							**A__      )    ->			Dict:
          self.training_tracker.update(
              state.global_step       ,							comment=F"Epoch {int(state.epoch      )}/{state.num_train_epochs}"       ,							force_update=A__      )
          _SCREAMING_SNAKE_CASE      =				None








'''simple docstring'''



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class    _a				(_lowerCamelCase  ,					unittest.TestCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  MobileBertTokenizer
       SCREAMING_SNAKE_CASE      =  MobileBertTokenizerFast
       SCREAMING_SNAKE_CASE      =  True
       SCREAMING_SNAKE_CASE      =  True
       SCREAMING_SNAKE_CASE      =  filter_non_english
       SCREAMING_SNAKE_CASE      =  'google/mobilebert-uncased'






       def        UpperCamelCase				(							self      )    ->			Any:
          super().setUp()

          _SCREAMING_SNAKE_CASE      =				[
              """[UNK]""",
              """[CLS]""",
              """[SEP]""",
              """[PAD]""",
              """[MASK]""",
              """want""",
              """##want""",
              """##ed""",
              """wa""",
              """un""",
              """runn""",
              """##ing""",
              """,""",
              """low""",
              """lowest""",
          ]
          _SCREAMING_SNAKE_CASE      =				os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["""vocab_file"""]      )
          with open(self.vocab_file       ,							"""w"""       ,							encoding="""utf-8"""      ) as vocab_writer:
             vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )

          _SCREAMING_SNAKE_CASE      =				[
              (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
              for tokenizer_def in self.tokenizers_list
          ]






       def        UpperCamelCase				(							self       ,							A__      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				"""UNwant\u00E9d,running"""
          _SCREAMING_SNAKE_CASE      =				"""unwanted, running"""
          return input_text, output_text






       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				self.tokenizer_class(self.vocab_file      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.tokenize("""UNwant\u00E9d,running"""      )
          self.assertListEqual(A__       ,							["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]      )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(A__      )       ,							[9, 6, 7, 12, 10, 11]      )






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          if not self.test_rust_tokenizer:
             return

          _SCREAMING_SNAKE_CASE      =				self.get_tokenizer()
          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer()

          _SCREAMING_SNAKE_CASE      =				"""UNwant\u00E9d,running"""

          _SCREAMING_SNAKE_CASE      =				tokenizer.tokenize(A__      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.tokenize(A__      )
          self.assertListEqual(A__       ,							A__      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.encode(A__       ,							add_special_tokens=A__      )
          self.assertListEqual(A__       ,							A__      )

          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer()
          _SCREAMING_SNAKE_CASE      =				tokenizer.encode(A__      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.encode(A__      )
          self.assertListEqual(A__       ,							A__      )

          # With lower casing
          _SCREAMING_SNAKE_CASE      =				self.get_tokenizer(do_lower_case=A__      )
          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer(do_lower_case=A__      )

          _SCREAMING_SNAKE_CASE      =				"""UNwant\u00E9d,running"""

          _SCREAMING_SNAKE_CASE      =				tokenizer.tokenize(A__      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.tokenize(A__      )
          self.assertListEqual(A__       ,							A__      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.encode(A__       ,							add_special_tokens=A__      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.encode(A__       ,							add_special_tokens=A__      )
          self.assertListEqual(A__       ,							A__      )

          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer()
          _SCREAMING_SNAKE_CASE      =				tokenizer.encode(A__      )
          _SCREAMING_SNAKE_CASE      =				rust_tokenizer.encode(A__      )
          self.assertListEqual(A__       ,							A__      )






       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer()

          self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""      )       ,							["""ah""", """\u535A""", """\u63A8""", """zz"""]      )






       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """      )       ,							["""hello""", """!""", """how""", """are""", """you""", """?"""]      )
          self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )       ,							["""hello"""]      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__       ,							strip_accents=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )       ,							["""hällo""", """!""", """how""", """are""", """you""", """?"""]      )
          self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )       ,							["""h\u00E9llo"""]      )






       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__       ,							strip_accents=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )       ,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]      )
          self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )       ,							["""hello"""]      )






       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )       ,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]      )
          self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""      )       ,							["""hello"""]      )






       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """      )       ,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )






       def        UpperCamelCase				(							self      )    ->			Dict:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__       ,							strip_accents=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )       ,							["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__       ,							strip_accents=A__      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """      )       ,							["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]      )






       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				BasicTokenizer(do_lower_case=A__       ,							never_split=["""[UNK]"""]      )

          self.assertListEqual(
              tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""      )       ,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]      )






       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

          _SCREAMING_SNAKE_CASE      =				{}
          for i, token in enumerate(A__      ):
             _SCREAMING_SNAKE_CASE      =				i
          _SCREAMING_SNAKE_CASE      =				WordpieceTokenizer(vocab=A__       ,							unk_token="""[UNK]"""      )

          self.assertListEqual(tokenizer.tokenize(""""""      )       ,							[]      )

          self.assertListEqual(tokenizer.tokenize("""unwanted running"""      )       ,							["""un""", """##want""", """##ed""", """runn""", """##ing"""]      )

          self.assertListEqual(tokenizer.tokenize("""unwantedX running"""      )       ,							["""[UNK]""", """runn""", """##ing"""]      )






       def        UpperCamelCase				(							self      )    ->			str:
          self.assertTrue(_is_whitespace(""" """      )      )
          self.assertTrue(_is_whitespace("""\t"""      )      )
          self.assertTrue(_is_whitespace("""\r"""      )      )
          self.assertTrue(_is_whitespace("""\n"""      )      )
          self.assertTrue(_is_whitespace("""\u00A0"""      )      )

          self.assertFalse(_is_whitespace("""A"""      )      )
          self.assertFalse(_is_whitespace("""-"""      )      )






       def        UpperCamelCase				(							self      )    ->			Union[str, Any]:
          self.assertTrue(_is_control("""\u0005"""      )      )

          self.assertFalse(_is_control("""A"""      )      )
          self.assertFalse(_is_control(""" """      )      )
          self.assertFalse(_is_control("""\t"""      )      )
          self.assertFalse(_is_control("""\r"""      )      )






       def        UpperCamelCase				(							self      )    ->			Dict:
          self.assertTrue(_is_punctuation("""-"""      )      )
          self.assertTrue(_is_punctuation("""$"""      )      )
          self.assertTrue(_is_punctuation("""`"""      )      )
          self.assertTrue(_is_punctuation("""."""      )      )

          self.assertFalse(_is_punctuation("""A"""      )      )
          self.assertFalse(_is_punctuation(""" """      )      )






       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				self.get_tokenizer()
          _SCREAMING_SNAKE_CASE      =				self.get_rust_tokenizer()

          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(A__      ) for t in ["""Test""", """\xad""", """test"""]]       ,							[["""[UNK]"""], [], ["""[UNK]"""]]      )

          self.assertListEqual(
              [rust_tokenizer.tokenize(A__      ) for t in ["""Test""", """\xad""", """test"""]]       ,							[["""[UNK]"""], [], ["""[UNK]"""]]      )






       @slow
       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.encode("""sequence builders"""       ,							add_special_tokens=A__      )
          _SCREAMING_SNAKE_CASE      =				tokenizer.encode("""multi-sequence build"""       ,							add_special_tokens=A__      )

          _SCREAMING_SNAKE_CASE      =				tokenizer.build_inputs_with_special_tokens(A__      )
          _SCREAMING_SNAKE_CASE      =				tokenizer.build_inputs_with_special_tokens(A__       ,							A__      )

          assert encoded_sentence == [1_01] + text + [1_02]
          assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






       def        UpperCamelCase				(							self      )    ->			List[str]:
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                _SCREAMING_SNAKE_CASE      =				self.rust_tokenizer_class.from_pretrained(A__       ,							**A__      )

                _SCREAMING_SNAKE_CASE      =				F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                _SCREAMING_SNAKE_CASE      =				tokenizer_r.encode_plus(
                    A__       ,							return_attention_mask=A__       ,							return_token_type_ids=A__       ,							return_offsets_mapping=A__       ,							add_special_tokens=A__       ,							)

                _SCREAMING_SNAKE_CASE      =				tokenizer_r.do_lower_case if hasattr(A__       ,							"""do_lower_case"""      ) else False
                _SCREAMING_SNAKE_CASE      =				(
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), """A"""),
                        ((1, 2), ""","""),
                        ((3, 5), """na"""),
                        ((5, 6), """##ï"""),
                        ((6, 8), """##ve"""),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), """Allen"""),
                        ((21, 23), """##NL"""),
                        ((23, 24), """##P"""),
                        ((25, 33), """sentence"""),
                        ((33, 34), """."""),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), """a"""),
                        ((1, 2), ""","""),
                        ((3, 8), """naive"""),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), """allen"""),
                        ((21, 23), """##nl"""),
                        ((23, 24), """##p"""),
                        ((25, 33), """sentence"""),
                        ((33, 34), """."""),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]       ,							tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]      )      )
                self.assertEqual([e[0] for e in expected_results]       ,							tokens["""offset_mapping"""]      )






       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				["""的""", """人""", """有"""]
          _SCREAMING_SNAKE_CASE      =				"""""".join(A__      )
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                _SCREAMING_SNAKE_CASE      =				True
                _SCREAMING_SNAKE_CASE      =				self.tokenizer_class.from_pretrained(A__       ,							**A__      )
                _SCREAMING_SNAKE_CASE      =				self.rust_tokenizer_class.from_pretrained(A__       ,							**A__      )

                _SCREAMING_SNAKE_CASE      =				tokenizer_p.encode(A__       ,							add_special_tokens=A__      )
                _SCREAMING_SNAKE_CASE      =				tokenizer_r.encode(A__       ,							add_special_tokens=A__      )

                _SCREAMING_SNAKE_CASE      =				tokenizer_r.convert_ids_to_tokens(A__      )
                _SCREAMING_SNAKE_CASE      =				tokenizer_p.convert_ids_to_tokens(A__      )

                # it is expected that each Chinese character is not preceded by "##"
                self.assertListEqual(A__       ,							A__      )
                self.assertListEqual(A__       ,							A__      )

                _SCREAMING_SNAKE_CASE      =				False
                _SCREAMING_SNAKE_CASE      =				self.rust_tokenizer_class.from_pretrained(A__       ,							**A__      )
                _SCREAMING_SNAKE_CASE      =				self.tokenizer_class.from_pretrained(A__       ,							**A__      )

                _SCREAMING_SNAKE_CASE      =				tokenizer_r.encode(A__       ,							add_special_tokens=A__      )
                _SCREAMING_SNAKE_CASE      =				tokenizer_p.encode(A__       ,							add_special_tokens=A__      )

                _SCREAMING_SNAKE_CASE      =				tokenizer_r.convert_ids_to_tokens(A__      )
                _SCREAMING_SNAKE_CASE      =				tokenizer_p.convert_ids_to_tokens(A__      )

                # it is expected that only the first Chinese character is not preceded by "##".
                _SCREAMING_SNAKE_CASE      =				[
                    F"##{token}" if idx != 0 else token for idx, token in enumerate(A__      )
                ]
                self.assertListEqual(A__       ,							A__      )
                self.assertListEqual(A__       ,							A__      )






