





'''simple docstring'''



import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				Tuple											=       CpmAntTokenizer
	a    :				Union[str, Any]											=       False


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				super().setUp()

				__lowercase										=							[
				    '''<d>''',
				    '''</d>''',
				    '''<s>''',
				    '''</s>''',
				    '''</_>''',
				    '''<unk>''',
				    '''<pad>''',
				    '''</n>''',
				    '''我''',
				    '''是''',
				    '''C''',
				    '''P''',
				    '''M''',
				    '''A''',
				    '''n''',
				    '''t''',
				]
				__lowercase										=							os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['''vocab_file'''] )
				with open(self.vocab_file     ,'''w'''     ,encoding='''utf-8''' ) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )


	@tooslow
	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b''' )
				__lowercase										=							'''今天天气真好！'''
				__lowercase										=							['''今天''', '''天气''', '''真''', '''好''', '''！''']
				__lowercase										=							tokenizer.tokenize(_lowerCamelCase )
				self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase )
				__lowercase										=							'''今天天气真好！'''
				__lowercase										=							[tokenizer.bos_token] + tokens

				__lowercase										=							[6, 9802, 14962, 2082, 831, 244]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase )     ,_lowerCamelCase )

				__lowercase										=							tokenizer.decode(_lowerCamelCase )
				self.assertEqual(_lowerCamelCase     ,_lowerCamelCase )












'''simple docstring'''



from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_SCREAMING_SNAKE_CASE					=    {'''configuration_van''': ['''VAN_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''VanConfig''']}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_SCREAMING_SNAKE_CASE					=    [
	    '''VAN_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''VanForImageClassification''',
	    '''VanModel''',
	    '''VanPreTrainedModel''',
	]

if TYPE_CHECKING:
	from .configuration_van import VAN_PRETRAINED_CONFIG_ARCHIVE_MAP, VanConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_van import (
		    VAN_PRETRAINED_MODEL_ARCHIVE_LIST,
		    VanForImageClassification,
		    VanModel,
		    VanPreTrainedModel,
		)

else:
	import sys

	_SCREAMING_SNAKE_CASE					=    _LazyModule(__name__, globals()['''__file__'''], _import_structure)





