





"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase_			=							{
    '''configuration_mobilebert''': [
        '''MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''MobileBertConfig''',
        '''MobileBertOnnxConfig''',
    ],
    '''tokenization_mobilebert''': ['''MobileBertTokenizer'''],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCAmelCase_			=							['''MobileBertTokenizerFast''']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCAmelCase_			=							[
        '''MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''MobileBertForMaskedLM''',
        '''MobileBertForMultipleChoice''',
        '''MobileBertForNextSentencePrediction''',
        '''MobileBertForPreTraining''',
        '''MobileBertForQuestionAnswering''',
        '''MobileBertForSequenceClassification''',
        '''MobileBertForTokenClassification''',
        '''MobileBertLayer''',
        '''MobileBertModel''',
        '''MobileBertPreTrainedModel''',
        '''load_tf_weights_in_mobilebert''',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCAmelCase_			=							[
        '''TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''TFMobileBertForMaskedLM''',
        '''TFMobileBertForMultipleChoice''',
        '''TFMobileBertForNextSentencePrediction''',
        '''TFMobileBertForPreTraining''',
        '''TFMobileBertForQuestionAnswering''',
        '''TFMobileBertForSequenceClassification''',
        '''TFMobileBertForTokenClassification''',
        '''TFMobileBertMainLayer''',
        '''TFMobileBertModel''',
        '''TFMobileBertPreTrainedModel''',
    ]


if TYPE_CHECKING:
    from .configuration_mobilebert import (
        MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
        MobileBertConfig,
        MobileBertOnnxConfig,
    )
    from .tokenization_mobilebert import MobileBertTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_mobilebert_fast import MobileBertTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mobilebert import (
            MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            MobileBertForMaskedLM,
            MobileBertForMultipleChoice,
            MobileBertForNextSentencePrediction,
            MobileBertForPreTraining,
            MobileBertForQuestionAnswering,
            MobileBertForSequenceClassification,
            MobileBertForTokenClassification,
            MobileBertLayer,
            MobileBertModel,
            MobileBertPreTrainedModel,
            load_tf_weights_in_mobilebert,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_mobilebert import (
            TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFMobileBertForMaskedLM,
            TFMobileBertForMultipleChoice,
            TFMobileBertForNextSentencePrediction,
            TFMobileBertForPreTraining,
            TFMobileBertForQuestionAnswering,
            TFMobileBertForSequenceClassification,
            TFMobileBertForTokenClassification,
            TFMobileBertMainLayer,
            TFMobileBertModel,
            TFMobileBertPreTrainedModel,
        )

else:
    import sys

    lowerCAmelCase_			=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)













"""simple docstring"""






import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_rjieba
@require_tokenizers
class 	lowerCAmelCase					(      snake_case			,	unittest.TestCase							):
       lowerCAmelCase__						 =  RoFormerTokenizer
       lowerCAmelCase__						 =  RoFormerTokenizerFast
       lowerCAmelCase__						 =  True
       lowerCAmelCase__						 =  True






       def 							__A (   self     ):
              super().setUp()






       def 							__A (   self    ,  **a__     ):
              return self.tokenizer_class.from_pretrained('junnyu/roformer_chinese_base'    ,  **a__     )






       def 							__A (   self    ,  **a__     ):
              return self.rust_tokenizer_class.from_pretrained('junnyu/roformer_chinese_base'    ,  **a__     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     '永和服装饰品有限公司,今天天气非常好'
              _UpperCAmelCase     =     '永和 服装 饰品 有限公司 , 今 天 天 气 非常 好'
              return input_text, output_text






       def 							__A (   self     ):
              _UpperCAmelCase     =     self.get_tokenizer()
              _UpperCAmelCase    ,							_UpperCAmelCase     =     self.get_chinese_input_output_texts()
              _UpperCAmelCase     =     tokenizer.tokenize(a__     )

              self.assertListEqual(a__    ,  output_text.split()     )

              _UpperCAmelCase     =     tokens + [tokenizer.unk_token]
              _UpperCAmelCase     =     [2_29_43, 2_13_32, 3_44_31, 4_59_04, 1_17, 3_06, 12_31, 12_31, 26_53, 3_39_94, 12_66, 1_00]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(a__     )    ,  a__     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     self.get_rust_tokenizer()
              _UpperCAmelCase    ,							_UpperCAmelCase     =     self.get_chinese_input_output_texts()
              _UpperCAmelCase     =     tokenizer.tokenize(a__     )
              self.assertListEqual(a__    ,  output_text.split()     )
              _UpperCAmelCase     =     tokens + [tokenizer.unk_token]
              _UpperCAmelCase     =     [2_29_43, 2_13_32, 3_44_31, 4_59_04, 1_17, 3_06, 12_31, 12_31, 26_53, 3_39_94, 12_66, 1_00]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(a__     )    ,  a__     )






       def 							__A (   self     ):
              pass






       def 							__A (   self     ):
              pass






       def 							__A (   self     ):
              pass






