





'''simple docstring'''



import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				Tuple											=       CpmAntTokenizer
	a    :				Union[str, Any]											=       False


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				super().setUp()

				__lowercase										=							[
				    '''<d>''',
				    '''</d>''',
				    '''<s>''',
				    '''</s>''',
				    '''</_>''',
				    '''<unk>''',
				    '''<pad>''',
				    '''</n>''',
				    '''我''',
				    '''是''',
				    '''C''',
				    '''P''',
				    '''M''',
				    '''A''',
				    '''n''',
				    '''t''',
				]
				__lowercase										=							os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['''vocab_file'''] )
				with open(self.vocab_file     ,'''w'''     ,encoding='''utf-8''' ) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )


	@tooslow
	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b''' )
				__lowercase										=							'''今天天气真好！'''
				__lowercase										=							['''今天''', '''天气''', '''真''', '''好''', '''！''']
				__lowercase										=							tokenizer.tokenize(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )
				__lowercase										=							'''今天天气真好！'''
				__lowercase										=							[tokenizer.bos_token] + tokens

				__lowercase										=							[6, 9802, 14962, 2082, 831, 244]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase )     ,_lowerCamelCase )

				__lowercase										=							tokenizer.decode(_lowerCamelCase )
				self.assertEqual(_lowerCamelCase     ,_lowerCamelCase )












'''simple docstring'''



# flake8: noqa
# Lint as: python3
_SCREAMING_SNAKE_CASE					=    [
    '''VerificationMode''',
    '''Version''',
    '''disable_progress_bar''',
    '''enable_progress_bar''',
    '''is_progress_bar_enabled''',
    '''experimental''',
]

from .info_utils import VerificationMode
from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
from .version import Version
from .experimental import experimental





