




"""simple docstring"""


import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class       lowercase(  __a ,    unittest.TestCase				):







 '''simple docstring'''





 lowercase__				      =      CpmAntTokenizer
 lowercase__				      =      False


 def 				UpperCamelCase_     (       self:	Optional[Any]	):




   '''simple docstring'''



   super().setUp()

   _snake_case							:	List[Any]			=						[
       """<d>""",
       """</d>""",
       """<s>""",
       """</s>""",
       """</_>""",
       """<unk>""",
       """<pad>""",
       """</n>""",
       """我""",
       """是""",
       """C""",
       """P""",
       """M""",
       """A""",
       """n""",
       """t""",
   ]
   _snake_case							:	Tuple			=						os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["""vocab_file"""]	)
   with open(self.vocab_file,   """w""",   encoding="""utf-8"""	) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)


 @tooslow
 def 				UpperCamelCase_     (       self:	Optional[Any]	):




   '''simple docstring'''



   _snake_case							:	Any			=						CpmAntTokenizer.from_pretrained("""openbmb/cpm-ant-10b"""	)
   _snake_case							:	Optional[Any]			=						"""今天天气真好！"""
   _snake_case							:	List[Any]			=						["""今天""", """天气""", """真""", """好""", """！"""]
   _snake_case							:	Tuple			=						tokenizer.tokenize(a_	)
   self.assertListEqual(a_,   a_	)
   _snake_case							:	List[Any]			=						"""今天天气真好！"""
   _snake_case							:	Union[str, Any]			=						[tokenizer.bos_token] + tokens

   _snake_case							:	Dict			=						[6, 9_802, 14_962, 2_082, 831, 244]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(a_	),   a_	)

   _snake_case							:	Dict			=						tokenizer.decode(a_	)
   self.assertEqual(a_,   a_	)








"""simple docstring"""


def       UpperCAmelCase__ (snake_case__							:      int					):
  """simple docstring"""

  if not isinstance(snake_case__	,     snake_case__					) or number < 0:
    raise ValueError("""Input must be a non-negative integer"""					)

  _snake_case							:	Dict			=						0
  while number:
    # This way we arrive at next set bit (next 1) instead of looping
    # through each bit and checking for 1s hence the
    # loop won't run 32 times it will only run the number of `1` times
    number &= number - 1
    count += 1
  return count


if __name__ == "__main__":
     import doctest

     doctest.testmod()


