



'''simple docstring'''



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class 	lowerCamelCase_							(					__lowercase    ,				unittest.TestCase			):
						_lowerCAmelCase     :   Any	     =				BertJapaneseTokenizer
						_lowerCAmelCase     :   Dict	     =				False
						_lowerCAmelCase     :   Optional[int]	     =				True







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													super().setUp()

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			[
													    "[UNK]",
													    "[CLS]",
													    "[SEP]",
													    "こんにちは",
													    "こん",
													    "にちは",
													    "ばんは",
													    "##こん",
													    "##にちは",
													    "##ばんは",
													    "世界",
													    "##世界",
													    "、",
													    "##、",
													    "。",
													    "##。",
													]

													SCREAMING_SNAKE_CASE					:      Tuple     					=			os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']				)
													with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''				) as vocab_writer:
																				vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)







						def 					__lowercase    (  self   :      str				,    lowerCAmelCase__   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			"こんにちは、世界。 \nこんばんは、世界。"
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			"こんにちは 、 世界 。 こんばんは 、 世界 。"
													return input_text, output_text







						def 					__lowercase    (  self   :      Dict				,    lowerCAmelCase__   :      Tuple				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			self.get_input_output_texts(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.decode(lowerCAmelCase__				,    clean_up_tokenization_spaces=lowerCAmelCase__				)
													return text, ids







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													pass  # TODO add if relevant







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													pass  # TODO add if relevant







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													pass  # TODO add if relevant







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			self.tokenizer_class(self.vocab_file				)

													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''				)
													self.assertListEqual(lowerCAmelCase__				,    ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']				)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)				,    [3, 12, 10, 14, 4, 9, 12, 10, 14]				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			self.tokenizer_class(self.vocab_file				,    word_tokenizer_type='''mecab'''				)
													self.assertIsNotNone(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			"こんにちは、世界。\nこんばんは、世界。"
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']				)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)				,    [3, 12, 10, 14, 4, 9, 12, 10, 14]				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			os.path.join(self.tmpdirname				,    '''tokenizer.bin'''				)
													with open(lowerCAmelCase__				,    '''wb'''				) as handle:
																				pickle.dump(lowerCAmelCase__				,    lowerCAmelCase__				)

													with open(lowerCAmelCase__				,    '''rb'''				) as handle:
																				SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			pickle.load(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer_new.tokenize(lowerCAmelCase__				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			MecabTokenizer(mecab_dic='''ipadic'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				,    )







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													try:
																				SCREAMING_SNAKE_CASE					:      int     					=			MecabTokenizer(mecab_dic='''unidic_lite'''				)
													except ModuleNotFoundError:
																				return

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				,    )







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													try:
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			MecabTokenizer(mecab_dic='''unidic'''				)
													except ModuleNotFoundError:
																				return

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				,    )







						def 					__lowercase    (  self   :      Tuple				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			MecabTokenizer(do_lower_case=lowerCAmelCase__				,    mecab_dic='''ipadic'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				,    )







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													try:
																				SCREAMING_SNAKE_CASE					:      Tuple     					=			MecabTokenizer(
																				    do_lower_case=lowerCAmelCase__				,    normalize_text=lowerCAmelCase__				,    mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''				)
													except RuntimeError:
																				# if dict doesn't exist in the system, previous code raises this error.
																				return

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				,    )







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			MecabTokenizer(normalize_text=lowerCAmelCase__				,    mecab_dic='''ipadic'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']				,    )







						@require_sudachi
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			self.tokenizer_class(self.vocab_file				,    word_tokenizer_type='''sudachi'''				)
													self.assertIsNotNone(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      str     					=			"こんにちは、世界。\nこんばんは、世界。"
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']				)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)				,    [3, 12, 10, 14, 4, 9, 12, 10, 14]				)

													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			os.path.join(self.tmpdirname				,    '''tokenizer.bin'''				)
													with open(lowerCAmelCase__				,    '''wb'''				) as handle:
																				pickle.dump(lowerCAmelCase__				,    lowerCAmelCase__				)

													with open(lowerCAmelCase__				,    '''rb'''				) as handle:
																				SCREAMING_SNAKE_CASE					:      Optional[int]     					=			pickle.load(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer_new.tokenize(lowerCAmelCase__				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						@require_sudachi
						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			SudachiTokenizer(sudachi_dict_type='''core'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']				,    )







						@require_sudachi
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			SudachiTokenizer(sudachi_dict_type='''core'''				,    sudachi_split_mode='''A'''				)

													self.assertListEqual(tokenizer.tokenize('''外国人参政権'''				)				,    ['''外国''', '''人''', '''参政''', '''権''']				)







						@require_sudachi
						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			SudachiTokenizer(sudachi_dict_type='''core'''				,    sudachi_split_mode='''B'''				)

													self.assertListEqual(tokenizer.tokenize('''外国人参政権'''				)				,    ['''外国人''', '''参政権''']				)







						@require_sudachi
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			SudachiTokenizer(sudachi_dict_type='''core'''				,    sudachi_split_mode='''C'''				)

													self.assertListEqual(tokenizer.tokenize('''外国人参政権'''				)				,    ['''外国人参政権''']				)







						@require_sudachi
						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			SudachiTokenizer(do_lower_case=lowerCAmelCase__				,    sudachi_dict_type='''core'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']				,    )







						@require_sudachi
						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			SudachiTokenizer(normalize_text=lowerCAmelCase__				,    sudachi_dict_type='''core'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']				,    )







						@require_sudachi
						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      int     					=			SudachiTokenizer(trim_whitespace=lowerCAmelCase__				,    sudachi_dict_type='''core'''				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				,    )







						@require_jumanpp
						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			self.tokenizer_class(self.vocab_file				,    word_tokenizer_type='''jumanpp'''				)
													self.assertIsNotNone(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			"こんにちは、世界。\nこんばんは、世界。"
													SCREAMING_SNAKE_CASE					:      Dict     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']				)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)				,    [3, 12, 10, 14, 4, 9, 12, 10, 14]				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			os.path.join(self.tmpdirname				,    '''tokenizer.bin'''				)
													with open(lowerCAmelCase__				,    '''wb'''				) as handle:
																				pickle.dump(lowerCAmelCase__				,    lowerCAmelCase__				)

													with open(lowerCAmelCase__				,    '''rb'''				) as handle:
																				SCREAMING_SNAKE_CASE					:      Any     					=			pickle.load(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer_new.tokenize(lowerCAmelCase__				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						@require_jumanpp
						def 					__lowercase    (  self   :      Tuple				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			JumanppTokenizer()

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				,    )







						@require_jumanpp
						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			JumanppTokenizer(do_lower_case=lowerCAmelCase__				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				,    )







						@require_jumanpp
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			JumanppTokenizer(normalize_text=lowerCAmelCase__				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				,    )







						@require_jumanpp
						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			JumanppTokenizer(trim_whitespace=lowerCAmelCase__				)

													self.assertListEqual(
													    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''				)				,    ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']				,    )







						@require_jumanpp
						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			JumanppTokenizer()

													self.assertListEqual(
													    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''				)				,    ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']				,    )







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

													SCREAMING_SNAKE_CASE					:      str     					=			{}
													for i, token in enumerate(lowerCAmelCase__				):
																				SCREAMING_SNAKE_CASE					:      Tuple     					=			i
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			WordpieceTokenizer(vocab=lowerCAmelCase__				,    unk_token='''[UNK]'''				)

													self.assertListEqual(tokenizer.tokenize(''''''				)				,    []				)

													self.assertListEqual(tokenizer.tokenize('''こんにちは'''				)				,    ['''こんにちは''']				)

													self.assertListEqual(tokenizer.tokenize('''こんばんは'''				)				,    ['''こん''', '''##ばんは''']				)

													self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''				)				,    ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']				)







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''				)
													SCREAMING_SNAKE_CASE					:      int     					=			tokenizer.subword_tokenizer

													SCREAMING_SNAKE_CASE					:      int     					=			subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''				)
													self.assertListEqual(lowerCAmelCase__				,    ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''				)
													self.assertListEqual(lowerCAmelCase__				,    ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']				)







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''				)

													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.encode('''ありがとう。'''				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      int     					=			tokenizer.encode('''どういたしまして。'''				,    add_special_tokens=lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				,    lowerCAmelCase__				)

													# 2 is for "[CLS]", 3 is for "[SEP]"
													assert encoded_sentence == [2] + text + [3]
													assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class 	lowerCamelCase_							(					__lowercase    ,				unittest.TestCase			):
						_lowerCAmelCase     :   Tuple	     =				BertJapaneseTokenizer
						_lowerCAmelCase     :   Dict	     =				False







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													super().setUp()

													SCREAMING_SNAKE_CASE					:      int     					=			["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']				)
													with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''				) as vocab_writer:
																				vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)







						def 					__lowercase    (  self   :      Dict				,    **lowerCAmelCase__   :      Any				):

													"""simple docstring"""
													return BertJapaneseTokenizer.from_pretrained(self.tmpdirname				,    subword_tokenizer_type='''character'''				,    **lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[Any]				,    lowerCAmelCase__   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      int     					=			"こんにちは、世界。 \nこんばんは、世界。"
													SCREAMING_SNAKE_CASE					:      Tuple     					=			"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
													return input_text, output_text







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													pass  # TODO add if relevant







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													pass  # TODO add if relevant







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													pass  # TODO add if relevant







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			self.tokenizer_class(self.vocab_file				,    subword_tokenizer_type='''character'''				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''				)
													self.assertListEqual(
													    lowerCAmelCase__				,    ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']				)
													self.assertListEqual(
													    tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)				,    [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]				)







						def 					__lowercase    (  self   :      Tuple				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			{}
													for i, token in enumerate(lowerCAmelCase__				):
																				SCREAMING_SNAKE_CASE					:      Optional[int]     					=			i
													SCREAMING_SNAKE_CASE					:      List[str]     					=			CharacterTokenizer(vocab=lowerCAmelCase__				,    unk_token='''[UNK]'''				)

													self.assertListEqual(tokenizer.tokenize(''''''				)				,    []				)

													self.assertListEqual(tokenizer.tokenize('''こんにちは'''				)				,    ['''こ''', '''ん''', '''に''', '''ち''', '''は''']				)

													self.assertListEqual(tokenizer.tokenize('''こんにちほ'''				)				,    ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			tokenizer.encode('''ありがとう。'''				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.encode('''どういたしまして。'''				,    add_special_tokens=lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				,    lowerCAmelCase__				)

													# 2 is for "[CLS]", 3 is for "[SEP]"
													assert encoded_sentence == [2] + text + [3]
													assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class 	lowerCamelCase_							(					unittest.TestCase			):







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			"cl-tohoku/bert-base-japanese"
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			AutoTokenizer.from_pretrained(lowerCAmelCase__				)
													self.assertIsInstance(lowerCAmelCase__				,    lowerCAmelCase__				)


class 	lowerCamelCase_							(					unittest.TestCase			):







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			"cl-tohoku/bert-base-japanese"
													with self.assertLogs('''transformers'''				,    level='''WARNING'''				) as cm:
																				BertTokenizer.from_pretrained(lowerCAmelCase__				)
																				self.assertTrue(
																				    cm.records[0].message.startswith(
																				        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																				        ''' is called from.'''				)				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			"bert-base-cased"
													with self.assertLogs('''transformers'''				,    level='''WARNING'''				) as cm:
																				BertJapaneseTokenizer.from_pretrained(lowerCAmelCase__				)
																				self.assertTrue(
																				    cm.records[0].message.startswith(
																				        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																				        ''' is called from.'''				)				)









'''simple docstring'''



from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import RegNetConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST, TFRegNetForImageClassification, TFRegNetModel


if is_vision_available():
							from PIL import Image

							from transformers import AutoImageProcessor
class 	lowerCamelCase_							:







						def __init__(  self   :      Optional[Any]				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      int=3				,    lowerCAmelCase__   :      Tuple=32				,    lowerCAmelCase__   :      Union[str, Any]=3				,    lowerCAmelCase__   :      Dict=10				,    lowerCAmelCase__   :      List[Any]=[10, 20, 30, 40]				,    lowerCAmelCase__   :      List[Any]=[1, 1, 2, 1]				,    lowerCAmelCase__   :      Optional[int]=True				,    lowerCAmelCase__   :      Any=True				,    lowerCAmelCase__   :      List[str]="relu"				,    lowerCAmelCase__   :      int=3				,    lowerCAmelCase__   :      Tuple=None				,    ):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			parent
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			batch_size
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			image_size
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			num_channels
													SCREAMING_SNAKE_CASE					:      str     					=			embeddings_size
													SCREAMING_SNAKE_CASE					:      str     					=			hidden_sizes
													SCREAMING_SNAKE_CASE					:      Tuple     					=			depths
													SCREAMING_SNAKE_CASE					:      Dict     					=			is_training
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			use_labels
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			hidden_act
													SCREAMING_SNAKE_CASE					:      str     					=			num_labels
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			scope
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			len(lowerCAmelCase__				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			None
													if self.use_labels:
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			ids_tensor([self.batch_size]				,    self.num_labels				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_config()
													return config, pixel_values, labels







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													return RegNetConfig(
													    num_channels=self.num_channels				,    embeddings_size=self.embeddings_size				,    hidden_sizes=self.hidden_sizes				,    depths=self.depths				,    hidden_act=self.hidden_act				,    num_labels=self.num_labels				,    )







						def 					__lowercase    (  self   :      Dict				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      int     					=			TFRegNetModel(config=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      int     					=			model(lowerCAmelCase__				,    training=lowerCAmelCase__				)
													# expected last hidden states: B, C, H // 32, W // 32
													self.parent.assertEqual(
													    result.last_hidden_state.shape				,    (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)				,    )







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      str				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			self.num_labels
													SCREAMING_SNAKE_CASE					:      Any     					=			TFRegNetForImageClassification(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Dict     					=			model(lowerCAmelCase__				,    labels=lowerCAmelCase__				,    training=lowerCAmelCase__				)
													self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.num_labels)				)







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      int     					=			self.prepare_config_and_inputs()
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      List[str]     					=			config_and_inputs
													SCREAMING_SNAKE_CASE					:      Tuple     					=			{'''pixel_values''': pixel_values}
													return config, inputs_dict


@require_tf
class 	lowerCamelCase_							(					snake_case_    ,				snake_case_    ,				unittest.TestCase			):
						_lowerCAmelCase     :   str	     =				(TFRegNetModel, TFRegNetForImageClassification) if is_tf_available() else ()
						_lowerCAmelCase     :   str	     =				(
						    {'feature-extraction': TFRegNetModel, 'image-classification': TFRegNetForImageClassification}
						    if is_tf_available()
						    else {}
						)

						_lowerCAmelCase     :   Tuple	     =				False
						_lowerCAmelCase     :   List[Any]	     =				False
						_lowerCAmelCase     :   List[str]	     =				False
						_lowerCAmelCase     :   str	     =				False
						_lowerCAmelCase     :   str	     =				False







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			TFRegNetModelTester(self				)
													SCREAMING_SNAKE_CASE					:      Dict     					=			ConfigTester(self				,    config_class=lowerCAmelCase__				,    has_text_modality=lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													return







						@unittest.skip(reason='''RegNet does not use inputs_embeds'''				)
						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													pass







						@unittest.skipIf(
						    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''				)				) == 0				,    reason='''TF does not support backprop for grouped convolutions on CPU.'''				,    )
						@slow
						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													super().test_keras_fit()







						@unittest.skip(reason='''RegNet does not support input and output embeddings'''				)
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																				SCREAMING_SNAKE_CASE					:      int     					=			model_class(lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Any     					=			inspect.signature(model.call				)
																				# signature.parameters is an OrderedDict => so arg_names order is deterministic
																				SCREAMING_SNAKE_CASE					:      Tuple     					=			[*signature.parameters.keys()]

																				SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''pixel_values''']
																				self.assertListEqual(arg_names[:1]				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													def check_hidden_states_output(lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Tuple				):
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			model_class(lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Dict     					=			model(**self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)				,    training=lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      int     					=			outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

																				SCREAMING_SNAKE_CASE					:      int     					=			self.model_tester.num_stages
																				self.assertEqual(len(lowerCAmelCase__				)				,    expected_num_stages + 1				)

																				# RegNet's feature maps are of shape (batch_size, num_channels, height, width)
																				self.assertListEqual(
																				    list(hidden_states[0].shape[-2:]				)				,    [self.model_tester.image_size // 2, self.model_tester.image_size // 2]				,    )

													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.model_tester.prepare_config_and_inputs_for_common()
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			['''basic''', '''bottleneck''']
													for model_class in self.all_model_classes:
																				for layer_type in layers_type:
																											SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			layer_type
																											SCREAMING_SNAKE_CASE					:      Tuple     					=			True
																											check_hidden_states_output(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)

																											# check that output_hidden_states also work using config
																											del inputs_dict["output_hidden_states"]
																											SCREAMING_SNAKE_CASE					:      Optional[int]     					=			True

																											check_hidden_states_output(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.model_tester.prepare_config_and_inputs_for_common()

													def check_equivalence(lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      str={}				):
																				SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			model(lowerCAmelCase__				,    return_dict=lowerCAmelCase__				,    **lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Any     					=			model(lowerCAmelCase__				,    return_dict=lowerCAmelCase__				,    **lowerCAmelCase__				).to_tuple()

																				def recursive_check(lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      str				):
																											if isinstance(lowerCAmelCase__				,    (List, Tuple)				):
																																		for tuple_iterable_value, dict_iterable_value in zip(lowerCAmelCase__				,    lowerCAmelCase__				):
																																									recursive_check(lowerCAmelCase__				,    lowerCAmelCase__				)
																											elif tuple_object is None:
																																		return
																											else:
																																		self.assertTrue(
																																		    all(tf.equal(lowerCAmelCase__				,    lowerCAmelCase__				)				)				,    msg=(
																																		        '''Tuple and dict output are not equal. Difference:'''
																																		        F""" {tf.math.reduce_max(tf.abs(tuple_object - dict_object				)				)}"""
																																		    )				,    )

																											recursive_check(lowerCAmelCase__				,    lowerCAmelCase__				)

													for model_class in self.all_model_classes:
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			model_class(lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      int     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      str     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      Tuple     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Dict     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    {'''output_hidden_states''': True}				)

																				SCREAMING_SNAKE_CASE					:      Any     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    {'''output_hidden_states''': True}				)







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_image_classification(*lowerCAmelCase__				)







						@slow
						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													for model_name in TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			TFRegNetModel.from_pretrained(lowerCAmelCase__				)
																				self.assertIsNotNone(lowerCAmelCase__				)


def 					UpperCAmelCase   (						):
							SCREAMING_SNAKE_CASE					:      List[Any]     					=			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''				)
							return image






@require_tf
@require_vision
class 	lowerCamelCase_							(					unittest.TestCase			):







						@cached_property
						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													return (
													    AutoImageProcessor.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]				)
													    if is_vision_available()
													    else None
													)







						@slow
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			TFRegNetForImageClassification.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			self.default_image_processor
													SCREAMING_SNAKE_CASE					:      int     					=			prepare_img()
													SCREAMING_SNAKE_CASE					:      List[str]     					=			image_processor(images=lowerCAmelCase__				,    return_tensors='''tf'''				)

													# forward pass
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			model(**lowerCAmelCase__				,    training=lowerCAmelCase__				)

													# verify the logits
													SCREAMING_SNAKE_CASE					:      List[str]     					=			tf.TensorShape((1, 10_00)				)
													self.assertEqual(outputs.logits.shape				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			tf.constant([-0.4180, -1.5051, -3.4836]				)

													tf.debugging.assert_near(outputs.logits[0, :3]				,    lowerCAmelCase__				,    atol=1e-4				)




