from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowercase											=      {
    '''configuration_bert''': ['''BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BertConfig''', '''BertOnnxConfig'''],
    '''tokenization_bert''': ['''BasicTokenizer''', '''BertTokenizer''', '''WordpieceTokenizer'''],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      ['''BertTokenizerFast''']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      [
        '''BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''BertForMaskedLM''',
        '''BertForMultipleChoice''',
        '''BertForNextSentencePrediction''',
        '''BertForPreTraining''',
        '''BertForQuestionAnswering''',
        '''BertForSequenceClassification''',
        '''BertForTokenClassification''',
        '''BertLayer''',
        '''BertLMHeadModel''',
        '''BertModel''',
        '''BertPreTrainedModel''',
        '''load_tf_weights_in_bert''',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      [
        '''TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''TFBertEmbeddings''',
        '''TFBertForMaskedLM''',
        '''TFBertForMultipleChoice''',
        '''TFBertForNextSentencePrediction''',
        '''TFBertForPreTraining''',
        '''TFBertForQuestionAnswering''',
        '''TFBertForSequenceClassification''',
        '''TFBertForTokenClassification''',
        '''TFBertLMHeadModel''',
        '''TFBertMainLayer''',
        '''TFBertModel''',
        '''TFBertPreTrainedModel''',
    ]
try:
    if not is_tensorflow_text_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      ['''TFBertTokenizer''']

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      [
        '''FlaxBertForCausalLM''',
        '''FlaxBertForMaskedLM''',
        '''FlaxBertForMultipleChoice''',
        '''FlaxBertForNextSentencePrediction''',
        '''FlaxBertForPreTraining''',
        '''FlaxBertForQuestionAnswering''',
        '''FlaxBertForSequenceClassification''',
        '''FlaxBertForTokenClassification''',
        '''FlaxBertModel''',
        '''FlaxBertPreTrainedModel''',
    ]

if TYPE_CHECKING:
    from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
    from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_bert_fast import BertTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_bert import (
            BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            BertForMaskedLM,
            BertForMultipleChoice,
            BertForNextSentencePrediction,
            BertForPreTraining,
            BertForQuestionAnswering,
            BertForSequenceClassification,
            BertForTokenClassification,
            BertLayer,
            BertLMHeadModel,
            BertModel,
            BertPreTrainedModel,
            load_tf_weights_in_bert,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_bert import (
            TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFBertEmbeddings,
            TFBertForMaskedLM,
            TFBertForMultipleChoice,
            TFBertForNextSentencePrediction,
            TFBertForPreTraining,
            TFBertForQuestionAnswering,
            TFBertForSequenceClassification,
            TFBertForTokenClassification,
            TFBertLMHeadModel,
            TFBertMainLayer,
            TFBertModel,
            TFBertPreTrainedModel,
        )

    try:
        if not is_tensorflow_text_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_bert_tf import TFBertTokenizer

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_bert import (
            FlaxBertForCausalLM,
            FlaxBertForMaskedLM,
            FlaxBertForMultipleChoice,
            FlaxBertForNextSentencePrediction,
            FlaxBertForPreTraining,
            FlaxBertForQuestionAnswering,
            FlaxBertForSequenceClassification,
            FlaxBertForTokenClassification,
            FlaxBertModel,
            FlaxBertPreTrainedModel,
        )

else:
    import sys

    __lowercase											=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin


@require_jieba
class 			lowerCamelCase_  (     UpperCAmelCase_		,			unittest.TestCase				):






    '''simple docstring'''






    a__			:		Any      			=   CpmAntTokenizer
    a__			:		Optional[Any]      			=   False







    def     UpperCamelCase__     (			self) ->			Any:
        super().setUp()

        __UpperCamelCase  :Optional[int]    						=	[
            '''<d>''',
            '''</d>''',
            '''<s>''',
            '''</s>''',
            '''</_>''',
            '''<unk>''',
            '''<pad>''',
            '''</n>''',
            '''我''',
            '''是''',
            '''C''',
            '''P''',
            '''M''',
            '''A''',
            '''n''',
            '''t''',
        ]
        __UpperCamelCase  :Optional[int]    						=	os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES['''vocab_file'''])
        with open(self.vocab_file				,				'''w'''				,				encoding='''utf-8''') as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))







    @tooslow
    def     UpperCamelCase__     (			self) ->			Dict:
        __UpperCamelCase  :Tuple    						=	CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b''')
        __UpperCamelCase  :Dict    						=	'''今天天气真好！'''
        __UpperCamelCase  :Tuple    						=	['''今天''', '''天气''', '''真''', '''好''', '''！''']
        __UpperCamelCase  :Optional[Any]    						=	tokenizer.tokenize(__lowercase)
        self.assertListEqual(__lowercase				,				__lowercase)
        __UpperCamelCase  :int    						=	'''今天天气真好！'''
        __UpperCamelCase  :List[str]    						=	[tokenizer.bos_token] + tokens

        __UpperCamelCase  :List[str]    						=	[6, 9_802, 14_962, 2_082, 831, 244]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase)				,				__lowercase)

        __UpperCamelCase  :Dict    						=	tokenizer.decode(__lowercase)
        self.assertEqual(__lowercase				,				__lowercase)





