



"""simple docstring"""

import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin







@require_jieba
class    __lowercase     (					_UpperCAmelCase					,    unittest.TestCase):







						"""simple docstring"""







						_A		:							int							       =		CpmAntTokenizer
						_A		:							List[Any]							       =		False







						def 						__UpperCamelCase							(self       ):
											super().setUp()

											snake_case_					:		Optional[int] 					=					[
											    """<d>""",
											    """</d>""",
											    """<s>""",
											    """</s>""",
											    """</_>""",
											    """<unk>""",
											    """<pad>""",
											    """</n>""",
											    """我""",
											    """是""",
											    """C""",
											    """P""",
											    """M""",
											    """A""",
											    """n""",
											    """t""",
											]
											snake_case_					:		Any 					=					os.path.join(self.tmpdirname   ,		VOCAB_FILES_NAMES["""vocab_file"""]       )
											with open(self.vocab_file   ,		"""w"""   ,		encoding="""utf-8"""       ) as vocab_writer:
																vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )







						@tooslow
						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Optional[int] 					=					CpmAntTokenizer.from_pretrained("""openbmb/cpm-ant-10b"""       )
											snake_case_					:		str 					=					"""今天天气真好！"""
											snake_case_					:		Dict 					=					["""今天""", """天气""", """真""", """好""", """！"""]
											snake_case_					:		Any 					=					tokenizer.tokenize(lowercase__       )
											self.assertListEqual(lowercase__   ,		lowercase__       )
											snake_case_					:		Dict 					=					"""今天天气真好！"""
											snake_case_					:		Union[str, Any] 					=					[tokenizer.bos_token] + tokens

											snake_case_					:		Union[str, Any] 					=					[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )   ,		lowercase__       )

											snake_case_					:		int 					=					tokenizer.decode(lowercase__       )
											self.assertEqual(lowercase__   ,		lowercase__       )









"""simple docstring"""

def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  int				,    SCREAMING_SNAKE_CASE__ :  int			):



					"""simple docstring"""





					return numa ^ numa < 0


if __name__ == "__main__":
			import doctest

			doctest.testmod()




