



'''simple docstring'''


import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin







@require_jieba
class 	UpperCAmelCase	(		__a					,	unittest.TestCase):


 '''simple docstring'''




 __magic_name__					: List[str]  =      CpmAntTokenizer
 __magic_name__					: Dict  =      False




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      super().setUp()

      a_       		=[
          "<d>",
          "</d>",
          "<s>",
          "</s>",
          "</_>",
          "<unk>",
          "<pad>",
          "</n>",
          "我",
          "是",
          "C",
          "P",
          "M",
          "A",
          "n",
          "t",
      ]
      a_       		=os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES["vocab_file"])
      with open(self.vocab_file							,  "w"							,  encoding="utf-8") as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))




 @tooslow
 def       lowercase_  (   self) ->      str:



      """simple docstring"""







      a_       		=CpmAntTokenizer.from_pretrained("openbmb/cpm-ant-10b")
      a_       		="今天天气真好！"
      a_       		=["今天", "天气", "真", "好", "！"]
      a_       		=tokenizer.tokenize(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)
      a_       		="今天天气真好！"
      a_       		=[tokenizer.bos_token] + tokens

      a_       		=[6, 9_8_0_2, 1_4_9_6_2, 2_0_8_2, 8_3_1, 2_4_4]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_)							,  lowerCAmelCase_)

      a_       		=tokenizer.decode(lowerCAmelCase_)
      self.assertEqual(lowerCAmelCase_							,  lowerCAmelCase_)







'''simple docstring'''


from .testing import (
    are_the_same_tensors,
    execute_subprocess_async,
    require_bnb,
    require_cpu,
    require_cuda,
    require_huggingface_suite,
    require_mps,
    require_multi_gpu,
    require_multi_xpu,
    require_safetensors,
    require_single_gpu,
    require_single_xpu,
    require_torch_min_version,
    require_tpu,
    require_xpu,
    skip,
    slow,
)
from .training import RegressionDataset, RegressionModel, RegressionModelaXPU


from .scripts import test_script, test_sync, test_ops  # isort: skip


