

"""simple docstring"""



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 					a						(       UpperCamelCase_,	unittest.TestCase				):



				"""simple docstring"""



				UpperCAmelCase						 =					BertJapaneseTokenizer
				UpperCAmelCase						 =					False
				UpperCAmelCase						 =					True

				def 				UpperCamelCase	(     self: List[Any]       ):






									"""simple docstring"""


									super().setUp()

									A__										=  [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''こんにちは''',
									    '''こん''',
									    '''にちは''',
									    '''ばんは''',
									    '''##こん''',
									    '''##にちは''',
									    '''##ばんは''',
									    '''世界''',
									    '''##世界''',
									    '''、''',
									    '''##、''',
									    '''。''',
									    '''##。''',
									]

									A__										=  os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["""vocab_file"""]       )
									with open(self.vocab_file    ,						"""w"""    ,						encoding="""utf-8"""       ) as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )

				def 				UpperCamelCase	(     self: List[Any]    ,						UpperCamelCase: int       ):






									"""simple docstring"""


									A__										=  '''こんにちは、世界。 \nこんばんは、世界。'''
									A__										=  '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
									return input_text, output_text

				def 				UpperCamelCase	(     self: Dict    ,						UpperCamelCase: Any       ):






									"""simple docstring"""


									A__										=  self.get_input_output_texts(UpperCamelCase__       )
									A__										=  tokenizer.encode(UpperCamelCase__    ,						add_special_tokens=UpperCamelCase__       )
									A__										=  tokenizer.decode(UpperCamelCase__    ,						clean_up_tokenization_spaces=UpperCamelCase__       )
									return text, ids

				def 				UpperCamelCase	(     self: int       ):






									"""simple docstring"""


									pass  # TODO add if relevant

				def 				UpperCamelCase	(     self: Tuple       ):






									"""simple docstring"""


									pass  # TODO add if relevant

				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									pass  # TODO add if relevant

				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class(self.vocab_file       )

									A__										=  tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""       )
									self.assertListEqual(UpperCamelCase__    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__       )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="""mecab"""       )
									self.assertIsNotNone(UpperCamelCase__       )

									A__										=  '''こんにちは、世界。\nこんばんは、世界。'''
									A__										=  tokenizer.tokenize(UpperCamelCase__       )
									self.assertListEqual(UpperCamelCase__    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__       )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

									A__										=  os.path.join(self.tmpdirname    ,						"""tokenizer.bin"""       )
									with open(UpperCamelCase__    ,						"""wb"""       ) as handle:
														pickle.dump(UpperCamelCase__    ,						UpperCamelCase__       )

									with open(UpperCamelCase__    ,						"""rb"""       ) as handle:
														A__										=  pickle.load(UpperCamelCase__       )

									A__										=  tokenizer_new.tokenize(UpperCamelCase__       )

									self.assertListEqual(UpperCamelCase__    ,						UpperCamelCase__       )

				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									A__										=  MecabTokenizer(mecab_dic="""ipadic"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)

				def 				UpperCamelCase	(     self: Dict       ):






									"""simple docstring"""


									try:
														A__										=  MecabTokenizer(mecab_dic="""unidic_lite"""       )
									except ModuleNotFoundError:
														return

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)

				def 				UpperCamelCase	(     self: Dict       ):






									"""simple docstring"""


									try:
														A__										=  MecabTokenizer(mecab_dic="""unidic"""       )
									except ModuleNotFoundError:
														return

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)

				def 				UpperCamelCase	(     self: Optional[Any]       ):






									"""simple docstring"""


									A__										=  MecabTokenizer(do_lower_case=UpperCamelCase__    ,						mecab_dic="""ipadic"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)

				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									try:
														A__										=  MecabTokenizer(
														    do_lower_case=UpperCamelCase__    ,						normalize_text=UpperCamelCase__    ,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""       )
									except RuntimeError:
														# if dict doesn't exist in the system, previous code raises this error.
														return

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)

				def 				UpperCamelCase	(     self: Optional[Any]       ):






									"""simple docstring"""


									A__										=  MecabTokenizer(normalize_text=UpperCamelCase__    ,						mecab_dic="""ipadic"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]    ,						)

				@require_sudachi
				def 				UpperCamelCase	(     self: int       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="""sudachi"""       )
									self.assertIsNotNone(UpperCamelCase__       )

									A__										=  '''こんにちは、世界。\nこんばんは、世界。'''
									A__										=  tokenizer.tokenize(UpperCamelCase__       )
									self.assertListEqual(UpperCamelCase__    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__       )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

									A__										=  os.path.join(self.tmpdirname    ,						"""tokenizer.bin"""       )
									with open(UpperCamelCase__    ,						"""wb"""       ) as handle:
														pickle.dump(UpperCamelCase__    ,						UpperCamelCase__       )

									with open(UpperCamelCase__    ,						"""rb"""       ) as handle:
														A__										=  pickle.load(UpperCamelCase__       )

									A__										=  tokenizer_new.tokenize(UpperCamelCase__       )

									self.assertListEqual(UpperCamelCase__    ,						UpperCamelCase__       )

				@require_sudachi
				def 				UpperCamelCase	(     self: Tuple       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(sudachi_dict_type="""core"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,						)

				@require_sudachi
				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(sudachi_dict_type="""core"""    ,						sudachi_split_mode="""A"""       )

									self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )    ,						["""外国""", """人""", """参政""", """権"""]       )

				@require_sudachi
				def 				UpperCamelCase	(     self: Optional[int]       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(sudachi_dict_type="""core"""    ,						sudachi_split_mode="""B"""       )

									self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )    ,						["""外国人""", """参政権"""]       )

				@require_sudachi
				def 				UpperCamelCase	(     self: Optional[int]       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(sudachi_dict_type="""core"""    ,						sudachi_split_mode="""C"""       )

									self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )    ,						["""外国人参政権"""]       )

				@require_sudachi
				def 				UpperCamelCase	(     self: int       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(do_lower_case=UpperCamelCase__    ,						sudachi_dict_type="""core"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,						)

				@require_sudachi
				def 				UpperCamelCase	(     self: List[Any]       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(normalize_text=UpperCamelCase__    ,						sudachi_dict_type="""core"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]    ,						)

				@require_sudachi
				def 				UpperCamelCase	(     self: Any       ):






									"""simple docstring"""


									A__										=  SudachiTokenizer(trim_whitespace=UpperCamelCase__    ,						sudachi_dict_type="""core"""       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,						)

				@require_jumanpp
				def 				UpperCamelCase	(     self: Any       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="""jumanpp"""       )
									self.assertIsNotNone(UpperCamelCase__       )

									A__										=  '''こんにちは、世界。\nこんばんは、世界。'''
									A__										=  tokenizer.tokenize(UpperCamelCase__       )
									self.assertListEqual(UpperCamelCase__    ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__       )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

									A__										=  os.path.join(self.tmpdirname    ,						"""tokenizer.bin"""       )
									with open(UpperCamelCase__    ,						"""wb"""       ) as handle:
														pickle.dump(UpperCamelCase__    ,						UpperCamelCase__       )

									with open(UpperCamelCase__    ,						"""rb"""       ) as handle:
														A__										=  pickle.load(UpperCamelCase__       )

									A__										=  tokenizer_new.tokenize(UpperCamelCase__       )

									self.assertListEqual(UpperCamelCase__    ,						UpperCamelCase__       )

				@require_jumanpp
				def 				UpperCamelCase	(     self: Optional[int]       ):






									"""simple docstring"""


									A__										=  JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)

				@require_jumanpp
				def 				UpperCamelCase	(     self: str       ):






									"""simple docstring"""


									A__										=  JumanppTokenizer(do_lower_case=UpperCamelCase__       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)

				@require_jumanpp
				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									A__										=  JumanppTokenizer(normalize_text=UpperCamelCase__       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,						)

				@require_jumanpp
				def 				UpperCamelCase	(     self: Optional[int]       ):






									"""simple docstring"""


									A__										=  JumanppTokenizer(trim_whitespace=UpperCamelCase__       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )    ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]    ,						)

				@require_jumanpp
				def 				UpperCamelCase	(     self: Dict       ):






									"""simple docstring"""


									A__										=  JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""       )    ,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]    ,						)

				def 				UpperCamelCase	(     self: Tuple       ):






									"""simple docstring"""


									A__										=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

									A__										=  {}
									for i, token in enumerate(UpperCamelCase__       ):
														A__										=  i
									A__										=  WordpieceTokenizer(vocab=UpperCamelCase__    ,						unk_token="""[UNK]"""       )

									self.assertListEqual(tokenizer.tokenize(""""""       )    ,						[]       )

									self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )    ,						["""こんにちは"""]       )

									self.assertListEqual(tokenizer.tokenize("""こんばんは"""       )    ,						["""こん""", """##ばんは"""]       )

									self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""       )    ,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]       )

				def 				UpperCamelCase	(     self: Optional[int]       ):






									"""simple docstring"""


									A__										=  BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""       )
									A__										=  tokenizer.subword_tokenizer

									A__										=  subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""       )
									self.assertListEqual(UpperCamelCase__    ,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]       )

									A__										=  subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""       )
									self.assertListEqual(UpperCamelCase__    ,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]       )





				def 				UpperCamelCase	(     self: Tuple       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""       )

									A__										=  tokenizer.encode("""ありがとう。"""    ,						add_special_tokens=UpperCamelCase__       )
									A__										=  tokenizer.encode("""どういたしまして。"""    ,						add_special_tokens=UpperCamelCase__       )

									A__										=  tokenizer.build_inputs_with_special_tokens(UpperCamelCase__       )
									A__										=  tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,						UpperCamelCase__       )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 					a						(       UpperCamelCase_,	unittest.TestCase				):



				"""simple docstring"""



				UpperCAmelCase						 =					BertJapaneseTokenizer
				UpperCAmelCase						 =					False

				def 				UpperCamelCase	(     self: Dict       ):






									"""simple docstring"""


									super().setUp()

									A__										=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

									A__										=  os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["""vocab_file"""]       )
									with open(self.vocab_file    ,						"""w"""    ,						encoding="""utf-8"""       ) as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )

				def 				UpperCamelCase	(     self: Optional[int]    ,						**UpperCamelCase: Tuple       ):






									"""simple docstring"""


									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="""character"""    ,						**UpperCamelCase__       )

				def 				UpperCamelCase	(     self: Optional[Any]    ,						UpperCamelCase: List[str]       ):






									"""simple docstring"""


									A__										=  '''こんにちは、世界。 \nこんばんは、世界。'''
									A__										=  '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
									return input_text, output_text

				def 				UpperCamelCase	(     self: Optional[int]       ):






									"""simple docstring"""


									pass  # TODO add if relevant

				def 				UpperCamelCase	(     self: str       ):






									"""simple docstring"""


									pass  # TODO add if relevant

				def 				UpperCamelCase	(     self: List[Any]       ):






									"""simple docstring"""


									pass  # TODO add if relevant

				def 				UpperCamelCase	(     self: int       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="""character"""       )

									A__										=  tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""       )
									self.assertListEqual(
									    UpperCamelCase__    ,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]       )
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(UpperCamelCase__       )    ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]       )

				def 				UpperCamelCase	(     self: List[str]       ):






									"""simple docstring"""


									A__										=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

									A__										=  {}
									for i, token in enumerate(UpperCamelCase__       ):
														A__										=  i
									A__										=  CharacterTokenizer(vocab=UpperCamelCase__    ,						unk_token="""[UNK]"""       )

									self.assertListEqual(tokenizer.tokenize(""""""       )    ,						[]       )

									self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )    ,						["""こ""", """ん""", """に""", """ち""", """は"""]       )

									self.assertListEqual(tokenizer.tokenize("""こんにちほ"""       )    ,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""]       )





				def 				UpperCamelCase	(     self: List[Any]       ):






									"""simple docstring"""


									A__										=  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""       )

									A__										=  tokenizer.encode("""ありがとう。"""    ,						add_special_tokens=UpperCamelCase__       )
									A__										=  tokenizer.encode("""どういたしまして。"""    ,						add_special_tokens=UpperCamelCase__       )

									A__										=  tokenizer.build_inputs_with_special_tokens(UpperCamelCase__       )
									A__										=  tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,						UpperCamelCase__       )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 					a						(       unittest.TestCase				):



				"""simple docstring"""



				def 				UpperCamelCase	(     self: int       ):






									"""simple docstring"""


									A__										=  '''cl-tohoku/bert-base-japanese'''
									A__										=  AutoTokenizer.from_pretrained(UpperCamelCase__       )
									self.assertIsInstance(UpperCamelCase__    ,						UpperCamelCase__       )



class 					a						(       unittest.TestCase				):



				"""simple docstring"""



				def 				UpperCamelCase	(     self: List[Any]       ):






									"""simple docstring"""


									A__										=  '''cl-tohoku/bert-base-japanese'''
									with self.assertLogs("""transformers"""    ,						level="""WARNING"""       ) as cm:
														BertTokenizer.from_pretrained(UpperCamelCase__       )
														self.assertTrue(
														    cm.records[0].message.startswith(
														        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
														        """ is called from."""       )       )
									A__										=  '''bert-base-cased'''
									with self.assertLogs("""transformers"""    ,						level="""WARNING"""       ) as cm:
														BertJapaneseTokenizer.from_pretrained(UpperCamelCase__       )
														self.assertTrue(
														    cm.records[0].message.startswith(
														        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
														        """ is called from."""       )       )









"""simple docstring"""



import baseaa





def  _snake_case     (    UpperCAmelCase_  :					str			):
     return baseaa.aaaencode(string.encode("""utf-8"""			)			)





def  _snake_case     (    UpperCAmelCase_  :					bytes			):
     return baseaa.aaadecode(UpperCAmelCase_			).decode("""utf-8"""			)


if __name__ == "__main__":
 import doctest

 doctest.testmod()






