import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class       lowercase__		(     UpperCamelCase_			,							unittest.TestCase):
						UpperCamelCase_  =	BertJapaneseTokenizer
						UpperCamelCase_  =	False
						UpperCamelCase_  =	True


						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												super().setUp()

												SCREAMING_SNAKE_CASE							:					Dict           =							[
												    '''[UNK]''',
												    '''[CLS]''',
												    '''[SEP]''',
												    '''こんにちは''',
												    '''こん''',
												    '''にちは''',
												    '''ばんは''',
												    '''##こん''',
												    '''##にちは''',
												    '''##ばんは''',
												    '''世界''',
												    '''##世界''',
												    '''、''',
												    '''##、''',
												    '''。''',
												    '''##。''',
												]

												SCREAMING_SNAKE_CASE							:					List[Any]           =							os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''vocab_file''']		)
												with open(self.vocab_file					,	'''w'''					,	encoding='''utf-8'''		) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)


						def  __A ( self			:       List[Any]					,	UpperCamelCase__			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							'''こんにちは、世界。 \nこんばんは、世界。'''
												SCREAMING_SNAKE_CASE							:					List[str]           =							'''こんにちは 、 世界 。 こんばんは 、 世界 。'''
												return input_text, output_text


						def  __A ( self			:       Dict					,	UpperCamelCase__			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE       ,						SCREAMING_SNAKE_CASE							:					List[str]           =							self.get_input_output_texts(UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Tuple           =							tokenizer.encode(UpperCamelCase__					,	add_special_tokens=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							tokenizer.decode(UpperCamelCase__					,	clean_up_tokenization_spaces=UpperCamelCase__		)
												return text, ids


						def  __A ( self			:       int		):





												'''simple docstring'''





												pass  # TODO add if relevant


						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												pass  # TODO add if relevant


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												pass  # TODO add if relevant


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							self.tokenizer_class(self.vocab_file		)

												SCREAMING_SNAKE_CASE							:					Dict           =							tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''		)
												self.assertListEqual(UpperCamelCase__					,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	[3, 12, 10, 14, 4, 9, 12, 10, 14]		)


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							self.tokenizer_class(self.vocab_file					,	word_tokenizer_type='''mecab'''		)
												self.assertIsNotNone(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							'''こんにちは、世界。\nこんばんは、世界。'''
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.tokenize(UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

												SCREAMING_SNAKE_CASE							:					List[str]           =							os.path.join(self.tmpdirname					,	'''tokenizer.bin'''		)
												with open(UpperCamelCase__					,	'''wb'''		) as handle:
																		pickle.dump(UpperCamelCase__					,	UpperCamelCase__		)

												with open(UpperCamelCase__					,	'''rb'''		) as handle:
																		SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							pickle.load(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					List[str]           =							tokenizer_new.tokenize(UpperCamelCase__		)

												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							MecabTokenizer(mecab_dic='''ipadic'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,	)


						def  __A ( self			:       Dict		):





												'''simple docstring'''





												try:
																		SCREAMING_SNAKE_CASE							:					List[str]           =							MecabTokenizer(mecab_dic='''unidic_lite'''		)
												except ModuleNotFoundError:
																		return

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,	)


						def  __A ( self			:       Dict		):





												'''simple docstring'''





												try:
																		SCREAMING_SNAKE_CASE							:					Optional[int]           =							MecabTokenizer(mecab_dic='''unidic'''		)
												except ModuleNotFoundError:
																		return

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,	)


						def  __A ( self			:       Optional[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[Any]           =							MecabTokenizer(do_lower_case=UpperCamelCase__					,	mecab_dic='''ipadic'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,	)


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												try:
																		SCREAMING_SNAKE_CASE							:					Dict           =							MecabTokenizer(
																		    do_lower_case=UpperCamelCase__					,	normalize_text=UpperCamelCase__					,	mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''		)
												except RuntimeError:
																		# if dict doesn't exist in the system, previous code raises this error.
																		return

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,	)


						def  __A ( self			:       Optional[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Any           =							MecabTokenizer(normalize_text=UpperCamelCase__					,	mecab_dic='''ipadic'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']					,	)


						@require_sudachi
						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							self.tokenizer_class(self.vocab_file					,	word_tokenizer_type='''sudachi'''		)
												self.assertIsNotNone(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					List[str]           =							'''こんにちは、世界。\nこんばんは、世界。'''
												SCREAMING_SNAKE_CASE							:					int           =							tokenizer.tokenize(UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							os.path.join(self.tmpdirname					,	'''tokenizer.bin'''		)
												with open(UpperCamelCase__					,	'''wb'''		) as handle:
																		pickle.dump(UpperCamelCase__					,	UpperCamelCase__		)

												with open(UpperCamelCase__					,	'''rb'''		) as handle:
																		SCREAMING_SNAKE_CASE							:					str           =							pickle.load(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer_new.tokenize(UpperCamelCase__		)

												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)


						@require_sudachi
						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							SudachiTokenizer(sudachi_dict_type='''core'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']					,	)


						@require_sudachi
						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							SudachiTokenizer(sudachi_dict_type='''core'''					,	sudachi_split_mode='''A'''		)

												self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)					,	['''外国''', '''人''', '''参政''', '''権''']		)


						@require_sudachi
						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							SudachiTokenizer(sudachi_dict_type='''core'''					,	sudachi_split_mode='''B'''		)

												self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)					,	['''外国人''', '''参政権''']		)


						@require_sudachi
						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							SudachiTokenizer(sudachi_dict_type='''core'''					,	sudachi_split_mode='''C'''		)

												self.assertListEqual(tokenizer.tokenize('''外国人参政権'''		)					,	['''外国人参政権''']		)


						@require_sudachi
						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							SudachiTokenizer(do_lower_case=UpperCamelCase__					,	sudachi_dict_type='''core'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']					,	)


						@require_sudachi
						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[str]           =							SudachiTokenizer(normalize_text=UpperCamelCase__					,	sudachi_dict_type='''core'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	[''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']					,	)


						@require_sudachi
						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							SudachiTokenizer(trim_whitespace=UpperCamelCase__					,	sudachi_dict_type='''core'''		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,	)


						@require_jumanpp
						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							self.tokenizer_class(self.vocab_file					,	word_tokenizer_type='''jumanpp'''		)
												self.assertIsNotNone(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					str           =							'''こんにちは、世界。\nこんばんは、世界。'''
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.tokenize(UpperCamelCase__		)
												self.assertListEqual(UpperCamelCase__					,	['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']		)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	[3, 12, 10, 14, 4, 9, 12, 10, 14]		)

												SCREAMING_SNAKE_CASE							:					List[str]           =							os.path.join(self.tmpdirname					,	'''tokenizer.bin'''		)
												with open(UpperCamelCase__					,	'''wb'''		) as handle:
																		pickle.dump(UpperCamelCase__					,	UpperCamelCase__		)

												with open(UpperCamelCase__					,	'''rb'''		) as handle:
																		SCREAMING_SNAKE_CASE							:					List[Any]           =							pickle.load(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					int           =							tokenizer_new.tokenize(UpperCamelCase__		)

												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)


						@require_jumanpp
						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							JumanppTokenizer()

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,	)


						@require_jumanpp
						def  __A ( self			:       str		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							JumanppTokenizer(do_lower_case=UpperCamelCase__		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,	)


						@require_jumanpp
						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							JumanppTokenizer(normalize_text=UpperCamelCase__		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,	)


						@require_jumanpp
						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							JumanppTokenizer(trim_whitespace=UpperCamelCase__		)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''		)					,	['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']					,	)


						@require_jumanpp
						def  __A ( self			:       Dict		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							JumanppTokenizer()

												self.assertListEqual(
												    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''		)					,	['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']					,	)


						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

												SCREAMING_SNAKE_CASE							:					Any           =							{}
												for i, token in enumerate(UpperCamelCase__		):
																		SCREAMING_SNAKE_CASE							:					Tuple           =							i
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							WordpieceTokenizer(vocab=UpperCamelCase__					,	unk_token='''[UNK]'''		)

												self.assertListEqual(tokenizer.tokenize(''''''		)					,	[]		)

												self.assertListEqual(tokenizer.tokenize('''こんにちは'''		)					,	['''こんにちは''']		)

												self.assertListEqual(tokenizer.tokenize('''こんばんは'''		)					,	['''こん''', '''##ばんは''']		)

												self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''		)					,	['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']		)


						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''		)
												SCREAMING_SNAKE_CASE							:					str           =							tokenizer.subword_tokenizer

												SCREAMING_SNAKE_CASE							:					Tuple           =							subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''		)
												self.assertListEqual(UpperCamelCase__					,	['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']		)

												SCREAMING_SNAKE_CASE							:					Tuple           =							subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''		)
												self.assertListEqual(UpperCamelCase__					,	['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']		)



						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[int]           =							self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''		)

												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer.encode('''ありがとう。'''					,	add_special_tokens=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.encode('''どういたしまして。'''					,	add_special_tokens=UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							tokenizer.build_inputs_with_special_tokens(UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.build_inputs_with_special_tokens(UpperCamelCase__					,	UpperCamelCase__		)

												# 2 is for "[CLS]", 3 is for "[SEP]"
												assert encoded_sentence == [2] + text + [3]
												assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class       lowercase__		(     UpperCamelCase_			,							unittest.TestCase):
						UpperCamelCase_  =	BertJapaneseTokenizer
						UpperCamelCase_  =	False


						def  __A ( self			:       Dict		):





												'''simple docstring'''





												super().setUp()

												SCREAMING_SNAKE_CASE							:					str           =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

												SCREAMING_SNAKE_CASE							:					List[Any]           =							os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''vocab_file''']		)
												with open(self.vocab_file					,	'''w'''					,	encoding='''utf-8'''		) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]		)		)


						def  __A ( self			:       Optional[int]					,	**UpperCamelCase__			:       Tuple		):





												'''simple docstring'''





												return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,	subword_tokenizer_type='''character'''					,	**UpperCamelCase__		)


						def  __A ( self			:       Optional[Any]					,	UpperCamelCase__			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Any           =							'''こんにちは、世界。 \nこんばんは、世界。'''
												SCREAMING_SNAKE_CASE							:					int           =							'''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
												return input_text, output_text


						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												pass  # TODO add if relevant


						def  __A ( self			:       str		):





												'''simple docstring'''





												pass  # TODO add if relevant


						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												pass  # TODO add if relevant


						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							self.tokenizer_class(self.vocab_file					,	subword_tokenizer_type='''character'''		)

												SCREAMING_SNAKE_CASE							:					Optional[int]           =							tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''		)
												self.assertListEqual(
												    UpperCamelCase__					,	['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']		)
												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(UpperCamelCase__		)					,	[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]		)


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							{}
												for i, token in enumerate(UpperCamelCase__		):
																		SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							i
												SCREAMING_SNAKE_CASE							:					Any           =							CharacterTokenizer(vocab=UpperCamelCase__					,	unk_token='''[UNK]'''		)

												self.assertListEqual(tokenizer.tokenize(''''''		)					,	[]		)

												self.assertListEqual(tokenizer.tokenize('''こんにちは'''		)					,	['''こ''', '''ん''', '''に''', '''ち''', '''は''']		)

												self.assertListEqual(tokenizer.tokenize('''こんにちほ'''		)					,	['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']		)



						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''		)

												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.encode('''ありがとう。'''					,	add_special_tokens=UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.encode('''どういたしまして。'''					,	add_special_tokens=UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.build_inputs_with_special_tokens(UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.build_inputs_with_special_tokens(UpperCamelCase__					,	UpperCamelCase__		)

												# 2 is for "[CLS]", 3 is for "[SEP]"
												assert encoded_sentence == [2] + text + [3]
												assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class       lowercase__		(     unittest.TestCase):


						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[Any]           =							'''cl-tohoku/bert-base-japanese'''
												SCREAMING_SNAKE_CASE							:					List[Any]           =							AutoTokenizer.from_pretrained(UpperCamelCase__		)
												self.assertIsInstance(UpperCamelCase__					,	UpperCamelCase__		)






class       lowercase__		(     unittest.TestCase):


						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							'''cl-tohoku/bert-base-japanese'''
												with self.assertLogs('''transformers'''					,	level='''WARNING'''		) as cm:
																		BertTokenizer.from_pretrained(UpperCamelCase__		)
																		self.assertTrue(
																		    cm.records[0].message.startswith(
																		        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																		        ''' is called from.'''		)		)
												SCREAMING_SNAKE_CASE							:					Dict           =							'''bert-base-cased'''
												with self.assertLogs('''transformers'''					,	level='''WARNING'''		) as cm:
																		BertJapaneseTokenizer.from_pretrained(UpperCamelCase__		)
																		self.assertTrue(
																		    cm.records[0].message.startswith(
																		        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																		        ''' is called from.'''		)		)






import itertools
import os
import random
import tempfile
import unittest

import numpy as np

from transformers import TvltFeatureExtractor, is_datasets_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
					import torch

if is_datasets_available():
					from datasets import load_dataset

__UpperCamelCase	: Dict    		=  random.Random()
def    A			(					_lowercase	,						_lowercase=1.0	,						_lowercase=None	,						_lowercase=None		):
						if rng is None:
												SCREAMING_SNAKE_CASE							:					Any           =							global_rng

						SCREAMING_SNAKE_CASE							:					int           =							[]
						for batch_idx in range(shape[0]		):
												values.append([]		)
												for _ in range(shape[1]		):
																		values[-1].append(rng.random() * scale		)

						return values





class       lowercase__		(     unittest.TestCase):


						def __init__( self			:       List[Any]					,	UpperCamelCase__			:       Optional[int]					,	UpperCamelCase__			:       str=7					,	UpperCamelCase__			:       Any=400					,	UpperCamelCase__			:       List[str]=2000					,	UpperCamelCase__			:       List[Any]=2048					,	UpperCamelCase__			:       Any=128					,	UpperCamelCase__			:       Union[str, Any]=1					,	UpperCamelCase__			:       List[Any]=512					,	UpperCamelCase__			:       str=30					,	UpperCamelCase__			:       Tuple=4_4100					,	):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							parent
												SCREAMING_SNAKE_CASE							:					str           =							batch_size
												SCREAMING_SNAKE_CASE							:					str           =							min_seq_length
												SCREAMING_SNAKE_CASE							:					Dict           =							max_seq_length
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							spectrogram_length
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							feature_size
												SCREAMING_SNAKE_CASE							:					Tuple           =							num_audio_channels
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							hop_length
												SCREAMING_SNAKE_CASE							:					List[Any]           =							chunk_length
												SCREAMING_SNAKE_CASE							:					str           =							sampling_rate


						def  __A ( self			:       Optional[Any]		):





												'''simple docstring'''





												return {
												    "spectrogram_length": self.spectrogram_length,
												    "feature_size": self.feature_size,
												    "num_audio_channels": self.num_audio_channels,
												    "hop_length": self.hop_length,
												    "chunk_length": self.chunk_length,
												    "sampling_rate": self.sampling_rate,
												}



						def  __A ( self			:       Tuple					,	UpperCamelCase__			:       List[str]=False					,	UpperCamelCase__			:       Optional[int]=False		):





												'''simple docstring'''





												def _flatten(UpperCamelCase__			:       str		):
																		return list(itertools.chain(*UpperCamelCase__		)		)

												if equal_length:
																		SCREAMING_SNAKE_CASE							:					List[str]           =							[floats_list((self.max_seq_length, self.feature_size)		) for _ in range(self.batch_size		)]
												else:
																		# make sure that inputs increase in size
																		SCREAMING_SNAKE_CASE							:					int           =							[
																		    floats_list((x, self.feature_size)		)
																		    for x in range(self.min_seq_length					,	self.max_seq_length					,	self.seq_length_diff		)
																		]
												if numpify:
																		SCREAMING_SNAKE_CASE							:					Optional[Any]           =							[np.asarray(UpperCamelCase__		) for x in speech_inputs]
												return speech_inputs






@require_torch
@require_torchaudio
class       lowercase__		(     UpperCamelCase_			,							unittest.TestCase):
						UpperCamelCase_  =	TvltFeatureExtractor


						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							TvltFeatureExtractionTester(self		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Any           =							self.feature_extraction_class(**self.feat_extract_dict		)
												self.assertTrue(hasattr(UpperCamelCase__					,	'''spectrogram_length'''		)		)
												self.assertTrue(hasattr(UpperCamelCase__					,	'''feature_size'''		)		)
												self.assertTrue(hasattr(UpperCamelCase__					,	'''num_audio_channels'''		)		)
												self.assertTrue(hasattr(UpperCamelCase__					,	'''hop_length'''		)		)
												self.assertTrue(hasattr(UpperCamelCase__					,	'''chunk_length'''		)		)
												self.assertTrue(hasattr(UpperCamelCase__					,	'''sampling_rate'''		)		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							self.feature_extraction_class(**self.feat_extract_dict		)

												with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE							:					Optional[Any]           =							feat_extract_first.save_pretrained(UpperCamelCase__		)[0]
																		check_json_file_has_correct_format(UpperCamelCase__		)
																		SCREAMING_SNAKE_CASE							:					List[Any]           =							self.feature_extraction_class.from_pretrained(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							feat_extract_first.to_dict()
												SCREAMING_SNAKE_CASE							:					str           =							feat_extract_second.to_dict()
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							dict_first.pop('''mel_filters'''		)
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							dict_second.pop('''mel_filters'''		)
												self.assertTrue(np.allclose(UpperCamelCase__					,	UpperCamelCase__		)		)
												self.assertEqual(UpperCamelCase__					,	UpperCamelCase__		)


						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							self.feature_extraction_class(**self.feat_extract_dict		)

												with tempfile.TemporaryDirectory() as tmpdirname:
																		SCREAMING_SNAKE_CASE							:					Tuple           =							os.path.join(UpperCamelCase__					,	'''feat_extract.json'''		)
																		feat_extract_first.to_json_file(UpperCamelCase__		)
																		SCREAMING_SNAKE_CASE							:					List[str]           =							self.feature_extraction_class.from_json_file(UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Any           =							feat_extract_first.to_dict()
												SCREAMING_SNAKE_CASE							:					str           =							feat_extract_second.to_dict()
												SCREAMING_SNAKE_CASE							:					int           =							dict_first.pop('''mel_filters'''		)
												SCREAMING_SNAKE_CASE							:					Any           =							dict_second.pop('''mel_filters'''		)
												self.assertTrue(np.allclose(UpperCamelCase__					,	UpperCamelCase__		)		)
												self.assertEqual(UpperCamelCase__					,	UpperCamelCase__		)


						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							self.feature_extraction_class(**self.feat_extract_dict		)

												# create three inputs of length 800, 1000, and 1200
												SCREAMING_SNAKE_CASE							:					List[Any]           =							[floats_list((1, x)		)[0] for x in range(800					,	1400					,	200		)]
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							[np.asarray(UpperCamelCase__		) for speech_input in speech_inputs]

												# Test not batched input
												SCREAMING_SNAKE_CASE							:					List[Any]           =							feature_extractor(np_speech_inputs[0]					,	return_tensors='''np'''					,	sampling_rate=4_4100		).audio_values

												self.assertTrue(encoded_audios.ndim == 4		)
												self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
												self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
												self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

												# Test batched
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							feature_extractor(UpperCamelCase__					,	return_tensors='''np'''					,	sampling_rate=4_4100		).audio_values

												self.assertTrue(encoded_audios.ndim == 4		)
												self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
												self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
												self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

												# Test audio masking
												SCREAMING_SNAKE_CASE							:					List[str]           =							feature_extractor(
												    UpperCamelCase__					,	return_tensors='''np'''					,	sampling_rate=4_4100					,	mask_audio=UpperCamelCase__		).audio_values

												self.assertTrue(encoded_audios.ndim == 4		)
												self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
												self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
												self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

												# Test 2-D numpy arrays are batched.
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							[floats_list((1, x)		)[0] for x in (800, 800, 800)]
												SCREAMING_SNAKE_CASE							:					Dict           =							np.asarray(UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					str           =							feature_extractor(UpperCamelCase__					,	return_tensors='''np'''					,	sampling_rate=4_4100		).audio_values
												self.assertTrue(encoded_audios.ndim == 4		)
												self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
												self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
												self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)


						def  __A ( self			:       Optional[int]					,	UpperCamelCase__			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[str]           =							load_dataset('''hf-internal-testing/librispeech_asr_dummy'''					,	'''clean'''					,	split='''validation'''		)
												# automatic decoding with librispeech
												SCREAMING_SNAKE_CASE							:					Dict           =							ds.sort('''id'''		).select(range(UpperCamelCase__		)		)[:num_samples]['''audio''']

												return [x["array"] for x in speech_samples]



						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							self._load_datasamples(1		)
												SCREAMING_SNAKE_CASE							:					int           =							TvltFeatureExtractor()
												SCREAMING_SNAKE_CASE							:					List[Any]           =							feature_extractor(UpperCamelCase__					,	return_tensors='''pt'''		).audio_values

												self.assertEquals(audio_values.shape					,	(1, 1, 192, 128)		)

												SCREAMING_SNAKE_CASE							:					str           =							torch.tensor([[-0.3032, -0.2708], [-0.4434, -0.4007]]		)
												self.assertTrue(torch.allclose(audio_values[0, 0, :2, :2]					,	UpperCamelCase__					,	atol=1E-4		)		)





