



import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class       _SCREAMING_SNAKE_CASE	(UpperCamelCase			,    unittest.TestCase ):
     lowerCAmelCase					=			RoCBertTokenizer
     lowerCAmelCase					=			None
     lowerCAmelCase					=			False
     lowerCAmelCase					=			True
     lowerCAmelCase					=			filter_non_english





     def 	__snake_case    (       self :							Union[str, Any]				)->List[Any]:
      super().setUp()

      __SCREAMING_SNAKE_CASE       :    Tuple       =	["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	{}
      __SCREAMING_SNAKE_CASE       :    Tuple       =	{}
      for i, value in enumerate(UpperCamelCase				):
       __SCREAMING_SNAKE_CASE       :    Tuple       =	i
       __SCREAMING_SNAKE_CASE       :    Dict       =	i
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"]				)
      __SCREAMING_SNAKE_CASE       :    Tuple       =	os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"]				)
      __SCREAMING_SNAKE_CASE       :    Tuple       =	os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"]				)
      with open(self.vocab_file     ,  "w"     ,  encoding="utf-8"				) as vocab_writer:
       vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)
      with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8"				) as word_shape_writer:
       json.dump(UpperCamelCase     ,  UpperCamelCase     ,  ensure_ascii=UpperCamelCase				)
      with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8"				) as word_pronunciation_writer:
       json.dump(UpperCamelCase     ,  UpperCamelCase     ,  ensure_ascii=UpperCamelCase				)





     def 	__snake_case    (       self :							List[str]				)->str:
      __SCREAMING_SNAKE_CASE       :    int       =	self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file				)

      __SCREAMING_SNAKE_CASE       :    Any       =	tokenizer.tokenize("你好[SEP]你是谁"				)
      self.assertListEqual(UpperCamelCase     ,  ["你", "好", "[SEP]", "你", "是", "谁"]				)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase				)     ,  [5, 6, 2, 5, 7, 8]				)
      self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase				)     ,  [5, 6, 2, 5, 7, 8]				)
      self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase				)     ,  [5, 6, 2, 5, 7, 8]				)





     def 	__snake_case    (       self :							Tuple				)->Any:
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	RoCBertBasicTokenizer()

      self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)     ,  ["ah", "\u535A", "\u63A8", "zz"]				)





     def 	__snake_case    (       self :							Union[str, Any]				)->str:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)     ,  ["hello", "!", "how", "are", "you", "?"]				)
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,  ["hello"]				)





     def 	__snake_case    (       self :							str				)->int:
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase     ,  strip_accents=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,  ["hällo", "!", "how", "are", "you", "?"]				)
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,  ["h\u00E9llo"]				)





     def 	__snake_case    (       self :							List[str]				)->str:
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase     ,  strip_accents=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,  ["hallo", "!", "how", "are", "you", "?"]				)
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,  ["hello"]				)





     def 	__snake_case    (       self :							Dict				)->Optional[Any]:
      __SCREAMING_SNAKE_CASE       :    List[str]       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,  ["hallo", "!", "how", "are", "you", "?"]				)
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,  ["hello"]				)





     def 	__snake_case    (       self :							List[Any]				)->List[Any]:
      __SCREAMING_SNAKE_CASE       :    Any       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"]				)





     def 	__snake_case    (       self :							List[Any]				)->Any:
      __SCREAMING_SNAKE_CASE       :    Any       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase     ,  strip_accents=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"]				)





     def 	__snake_case    (       self :							Optional[Any]				)->Optional[int]:
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase     ,  strip_accents=UpperCamelCase				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"]				)





     def 	__snake_case    (       self :							Dict				)->Dict:
      __SCREAMING_SNAKE_CASE       :    Tuple       =	RoCBertBasicTokenizer(do_lower_case=UpperCamelCase     ,  never_split=["[UNK]"]				)

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)





     def 	__snake_case    (       self :							List[str]				)->str:
      __SCREAMING_SNAKE_CASE       :    str       =	["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	{}
      for i, token in enumerate(UpperCamelCase				):
       __SCREAMING_SNAKE_CASE       :    int       =	i
      __SCREAMING_SNAKE_CASE       :    Any       =	RoCBertWordpieceTokenizer(vocab=UpperCamelCase     ,  unk_token="[UNK]"				)

      self.assertListEqual(tokenizer.tokenize(""				)     ,  []				)

      self.assertListEqual(tokenizer.tokenize("unwanted running"				)     ,  ["un", "##want", "##ed", "runn", "##ing"]				)

      self.assertListEqual(tokenizer.tokenize("unwantedX running"				)     ,  ["[UNK]", "runn", "##ing"]				)





     def 	__snake_case    (       self :							List[str]				)->Tuple:
      self.assertTrue(_is_whitespace(" "				)				)
      self.assertTrue(_is_whitespace("\t"				)				)
      self.assertTrue(_is_whitespace("\r"				)				)
      self.assertTrue(_is_whitespace("\n"				)				)
      self.assertTrue(_is_whitespace("\u00A0"				)				)

      self.assertFalse(_is_whitespace("A"				)				)
      self.assertFalse(_is_whitespace("-"				)				)





     def 	__snake_case    (       self :							Any				)->Optional[Any]:
      self.assertTrue(_is_control("\u0005"				)				)

      self.assertFalse(_is_control("A"				)				)
      self.assertFalse(_is_control(" "				)				)
      self.assertFalse(_is_control("\t"				)				)
      self.assertFalse(_is_control("\r"				)				)





     def 	__snake_case    (       self :							List[Any]				)->Any:
      self.assertTrue(_is_punctuation("-"				)				)
      self.assertTrue(_is_punctuation("$"				)				)
      self.assertTrue(_is_punctuation("`"				)				)
      self.assertTrue(_is_punctuation("."				)				)

      self.assertFalse(_is_punctuation("A"				)				)
      self.assertFalse(_is_punctuation(" "				)				)





     def 	__snake_case    (       self :							Tuple				)->Any:
      __SCREAMING_SNAKE_CASE       :    List[str]       =	self.get_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(UpperCamelCase				) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]]				)

      if self.test_rust_tokenizer:
       __SCREAMING_SNAKE_CASE       :    Any       =	self.get_rust_tokenizer()
       self.assertListEqual(
           [rust_tokenizer.tokenize(UpperCamelCase				) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]]				)





     def 	__snake_case    (       self :							Any				)->Tuple:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
        __SCREAMING_SNAKE_CASE       :    Optional[int]       =	self.rust_tokenizer_class.from_pretrained(UpperCamelCase     ,  **UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    List[str]       =	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
        __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	tokenizer_r.encode_plus(
            UpperCamelCase     ,  return_attention_mask=UpperCamelCase     ,  return_token_type_ids=UpperCamelCase     ,  return_offsets_mapping=UpperCamelCase     ,  add_special_tokens=UpperCamelCase     ,  )

        __SCREAMING_SNAKE_CASE       :    Optional[int]       =	tokenizer_r.do_lower_case if hasattr(UpperCamelCase     ,  "do_lower_case"				) else False
        __SCREAMING_SNAKE_CASE       :    Any       =	(
            [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), "A"),
                ((1, 2), ","),
                ((3, 5), "na"),
                ((5, 6), "##ï"),
                ((6, 8), "##ve"),
                ((9, 1_5), tokenizer_r.mask_token),
                ((1_6, 2_1), "Allen"),
                ((2_1, 2_3), "##NL"),
                ((2_3, 2_4), "##P"),
                ((2_5, 3_3), "sentence"),
                ((3_3, 3_4), "."),
                ((0, 0), tokenizer_r.sep_token),
            ]
            if not do_lower_case
            else [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), "a"),
                ((1, 2), ","),
                ((3, 8), "naive"),
                ((9, 1_5), tokenizer_r.mask_token),
                ((1_6, 2_1), "allen"),
                ((2_1, 2_3), "##nl"),
                ((2_3, 2_4), "##p"),
                ((2_5, 3_3), "sentence"),
                ((3_3, 3_4), "."),
                ((0, 0), tokenizer_r.sep_token),
            ]
        )

        self.assertEqual(
            [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
        self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"]				)





     def 	__snake_case    (       self :							Union[str, Any]				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	["的", "人", "有"]
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	"".join(UpperCamelCase				)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
        __SCREAMING_SNAKE_CASE       :    Dict       =	True
        __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	self.tokenizer_class.from_pretrained(UpperCamelCase     ,  **UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    Tuple       =	self.rust_tokenizer_class.from_pretrained(UpperCamelCase     ,  **UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    Any       =	tokenizer_p.encode(UpperCamelCase     ,  add_special_tokens=UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	tokenizer_r.encode(UpperCamelCase     ,  add_special_tokens=UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    Tuple       =	tokenizer_r.convert_ids_to_tokens(UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    str       =	tokenizer_p.convert_ids_to_tokens(UpperCamelCase				)

        # it is expected that each Chinese character is not preceded by "##"
        self.assertListEqual(UpperCamelCase     ,  UpperCamelCase				)
        self.assertListEqual(UpperCamelCase     ,  UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    Tuple       =	False
        __SCREAMING_SNAKE_CASE       :    List[str]       =	self.rust_tokenizer_class.from_pretrained(UpperCamelCase     ,  **UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	self.tokenizer_class.from_pretrained(UpperCamelCase     ,  **UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    List[Any]       =	tokenizer_r.encode(UpperCamelCase     ,  add_special_tokens=UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    int       =	tokenizer_p.encode(UpperCamelCase     ,  add_special_tokens=UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    Optional[int]       =	tokenizer_r.convert_ids_to_tokens(UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	tokenizer_p.convert_ids_to_tokens(UpperCamelCase				)

        # it is expected that only the first Chinese character is not preceded by "##".
        __SCREAMING_SNAKE_CASE       :    Optional[int]       =	[
            F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCamelCase				)
        ]
        self.assertListEqual(UpperCamelCase     ,  UpperCamelCase				)
        self.assertListEqual(UpperCamelCase     ,  UpperCamelCase				)





     @slow
     def 	__snake_case    (       self :							List[Any]				)->Tuple:
      __SCREAMING_SNAKE_CASE       :    str       =	self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file				)

      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase				)
      __SCREAMING_SNAKE_CASE       :    Optional[int]       =	tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase				)

      __SCREAMING_SNAKE_CASE       :    int       =	tokenizer.build_inputs_with_special_tokens(UpperCamelCase				)
      __SCREAMING_SNAKE_CASE       :    List[Any]       =	tokenizer.build_inputs_with_special_tokens(UpperCamelCase     ,  UpperCamelCase				)

      assert encoded_sentence == [1] + text + [2]
      assert encoded_pair == [1] + text + [2] + text_a + [2]






     def 	__snake_case    (       self :							Union[str, Any]				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	self.get_tokenizers(do_lower_case=UpperCamelCase				)
      for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""				):
        __SCREAMING_SNAKE_CASE       :    Dict       =	"你好，你是谁"
        __SCREAMING_SNAKE_CASE       :    List[Any]       =	tokenizer.tokenize(UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    List[str]       =	tokenizer.convert_tokens_to_ids(UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	tokenizer.convert_tokens_to_shape_ids(UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    int       =	tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase				)
        __SCREAMING_SNAKE_CASE       :    int       =	tokenizer.prepare_for_model(
            UpperCamelCase     ,  UpperCamelCase     ,  UpperCamelCase     ,  add_special_tokens=UpperCamelCase				)

        __SCREAMING_SNAKE_CASE       :    Tuple       =	tokenizer.encode_plus(UpperCamelCase     ,  add_special_tokens=UpperCamelCase				)

        self.assertEqual(UpperCamelCase     ,  UpperCamelCase				)











import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment

@require_huggingface_suite
class       _SCREAMING_SNAKE_CASE	(unittest.TestCase ):





     def 	__snake_case    (       self :							Optional[Any]				)->Union[str, Any]:
      __SCREAMING_SNAKE_CASE       :    Optional[Any]       =	inspect.getfile(accelerate.test_utils				)
      __SCREAMING_SNAKE_CASE       :    Union[str, Any]       =	os.path.sep.join(
          mod_file.split(os.path.sep				)[:-1] + ["scripts", "external_deps", "test_metrics.py"]				)

      from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

      __SCREAMING_SNAKE_CASE       :    Tuple       =	test_metrics





     @require_cpu
     def 	__snake_case    (       self :							Union[str, Any]				)->List[Any]:
      debug_launcher(self.test_metrics.main     ,  num_processes=1				)





     @require_cpu
     def 	__snake_case    (       self :							str				)->List[str]:
      debug_launcher(self.test_metrics.main				)





     @require_single_gpu
     def 	__snake_case    (       self :							Union[str, Any]				)->str:
      self.test_metrics.main()






     @require_multi_gpu
     def 	__snake_case    (       self :							List[str]				)->int:
      print(F"""Found {torch.cuda.device_count()} devices."""				)
      __SCREAMING_SNAKE_CASE       :    Any       =	["torchrun", F"""--nproc_per_node={torch.cuda.device_count()}""", self.test_file_path]
      with patch_environment(omp_num_threads=1				):
       execute_subprocess_async(UpperCamelCase     ,  env=os.environ.copy()				)






