



"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCAmelCase       :					Dict      					=					{
    '''configuration_mgp_str''': ['''MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MgpstrConfig'''],
    '''processing_mgp_str''': ['''MgpstrProcessor'''],
    '''tokenization_mgp_str''': ['''MgpstrTokenizer'''],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __lowerCAmelCase       :					Dict      					=					[
         '''MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''MgpstrModel''',
         '''MgpstrPreTrainedModel''',
         '''MgpstrForSceneTextRecognition''',
     ]

if TYPE_CHECKING:
     from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig
     from .processing_mgp_str import MgpstrProcessor
     from .tokenization_mgp_str import MgpstrTokenizer

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_mgp_str import (
              MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST,
              MgpstrForSceneTextRecognition,
              MgpstrModel,
              MgpstrPreTrainedModel,
          )
else:
     import sys

     __lowerCAmelCase       :					Optional[Any]      					=					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






"""simple docstring"""





import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_rjieba
@require_tokenizers
class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__  ,				unittest.TestCase		):





		"""simple docstring"""






		_lowerCamelCase			  =       RoFormerTokenizer
		_lowerCamelCase			  =       RoFormerTokenizerFast
		_lowerCamelCase			  =       True
		_lowerCamelCase			  =       True





		def    UpperCAmelCase__							(					self   ) ->						Any:







								'''simple docstring'''

								super().setUp()





		def    UpperCAmelCase__							(					self							,    **_lowercase   ) ->						str:







								'''simple docstring'''

								return self.tokenizer_class.from_pretrained("""junnyu/roformer_chinese_base"""							,    **_lowercase   )





		def    UpperCAmelCase__							(					self							,    **_lowercase   ) ->						Union[str, Any]:







								'''simple docstring'''

								return self.rust_tokenizer_class.from_pretrained("""junnyu/roformer_chinese_base"""							,    **_lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								snake_case_   : Tuple						=			"""永和服装饰品有限公司,今天天气非常好"""
								snake_case_   : int						=			"""永和 服装 饰品 有限公司 , 今 天 天 气 非常 好"""
								return input_text, output_text





		def    UpperCAmelCase__							(					self   ) ->						str:







								'''simple docstring'''

								snake_case_   : List[str]						=			self.get_tokenizer()
								snake_case_			,		snake_case_   : Optional[Any]						=			self.get_chinese_input_output_texts()
								snake_case_   : List[str]						=			tokenizer.tokenize(_lowercase   )

								self.assertListEqual(_lowercase							,    output_text.split()   )

								snake_case_   : str						=			tokens + [tokenizer.unk_token]
								snake_case_   : Tuple						=			[2_2_9_4_3, 2_1_3_3_2, 3_4_4_3_1, 4_5_9_0_4, 1_1_7, 3_0_6, 1_2_3_1, 1_2_3_1, 2_6_5_3, 3_3_9_9_4, 1_2_6_6, 1_0_0]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase   )							,    _lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						Any:







								'''simple docstring'''

								snake_case_   : int						=			self.get_rust_tokenizer()
								snake_case_			,		snake_case_   : List[Any]						=			self.get_chinese_input_output_texts()
								snake_case_   : Union[str, Any]						=			tokenizer.tokenize(_lowercase   )
								self.assertListEqual(_lowercase							,    output_text.split()   )
								snake_case_   : Optional[int]						=			tokens + [tokenizer.unk_token]
								snake_case_   : Union[str, Any]						=			[2_2_9_4_3, 2_1_3_3_2, 3_4_4_3_1, 4_5_9_0_4, 1_1_7, 3_0_6, 1_2_3_1, 1_2_3_1, 2_6_5_3, 3_3_9_9_4, 1_2_6_6, 1_0_0]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase   )							,    _lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						Tuple:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						Optional[Any]:







								'''simple docstring'''

								pass





		def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







								'''simple docstring'''

								pass

