




import inspect
import unittest

from transformers import ViTHybridConfig
from transformers.testing_utils import require_accelerate, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch
		from torch import nn

		from transformers import ViTHybridForImageClassification, ViTHybridImageProcessor, ViTHybridModel
		from transformers.models.vit_hybrid.modeling_vit_hybrid import VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
		from PIL import Image

class 							__UpperCamelCase			:





				'''simple docstring'''



				def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_=13       , UpperCAmelCase_=64       , UpperCAmelCase_=2       , UpperCAmelCase_=3       , UpperCAmelCase_=True       , UpperCAmelCase_=True       , UpperCAmelCase_=32       , UpperCAmelCase_=5       , UpperCAmelCase_=4       , UpperCAmelCase_=37       , UpperCAmelCase_="gelu"       , UpperCAmelCase_=0.1       , UpperCAmelCase_=0.1       , UpperCAmelCase_=10       , UpperCAmelCase_=0.02       , UpperCAmelCase_=[1, 16, 4, 4]       , UpperCAmelCase_=None       , ):
											lowerCAmelCase   =		parent
											lowerCAmelCase   =		batch_size
											lowerCAmelCase   =		image_size
											lowerCAmelCase   =		patch_size
											lowerCAmelCase   =		num_channels
											lowerCAmelCase   =		is_training
											lowerCAmelCase   =		use_labels
											lowerCAmelCase   =		hidden_size
											lowerCAmelCase   =		num_hidden_layers
											lowerCAmelCase   =		num_attention_heads
											lowerCAmelCase   =		intermediate_size
											lowerCAmelCase   =		hidden_act
											lowerCAmelCase   =		hidden_dropout_prob
											lowerCAmelCase   =		attention_probs_dropout_prob
											lowerCAmelCase   =		type_sequence_label_size
											lowerCAmelCase   =		initializer_range
											lowerCAmelCase   =		scope
											lowerCAmelCase   =		backbone_featmap_shape

											# in ViT hybrid, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
											# the number of patches is based on the feature map of the backbone, which by default uses an output stride
											# of 32, which means that the feature map has a spatial resolution of 1/32 of the input image size
											lowerCAmelCase   =		(self.image_size // 32) ** 2
											lowerCAmelCase   =		num_patches + 1



				def       __snake_case		( self						):
											lowerCAmelCase   =		floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

											lowerCAmelCase   =		None
											if self.use_labels:
																		lowerCAmelCase   =		ids_tensor([self.batch_size]       , self.type_sequence_label_size						)

											lowerCAmelCase   =		self.get_config()

											return config, pixel_values, labels



				def       __snake_case		( self						):
											lowerCAmelCase   =		{
											    '''global_padding''': '''same''',
											    '''layer_type''': '''bottleneck''',
											    '''depths''': [3, 4, 9],
											    '''out_features''': ['''stage1''', '''stage2''', '''stage3'''],
											    '''embedding_dynamic_padding''': True,
											    '''hidden_sizes''': [4, 8, 16, 32],
											    '''num_groups''': 2,
											}

											return ViTHybridConfig(
											    image_size=self.image_size       , patch_size=self.patch_size       , num_channels=self.num_channels       , hidden_size=self.hidden_size       , num_hidden_layers=self.num_hidden_layers       , num_attention_heads=self.num_attention_heads       , intermediate_size=self.intermediate_size       , hidden_act=self.hidden_act       , hidden_dropout_prob=self.hidden_dropout_prob       , attention_probs_dropout_prob=self.attention_probs_dropout_prob       , is_decoder=__UpperCamelCase       , initializer_range=self.initializer_range       , backbone_featmap_shape=self.backbone_featmap_shape       , backbone_config=__UpperCamelCase       , )



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_						):
											lowerCAmelCase   =		ViTHybridModel(config=__UpperCamelCase						)
											model.to(__UpperCamelCase						)
											model.eval()
											lowerCAmelCase   =		model(__UpperCamelCase						)
											self.parent.assertEqual(result.last_hidden_state.shape       , (self.batch_size, self.seq_length, self.hidden_size)						)



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_						):
											lowerCAmelCase   =		self.type_sequence_label_size
											lowerCAmelCase   =		ViTHybridForImageClassification(__UpperCamelCase						)
											model.to(__UpperCamelCase						)
											model.eval()
											lowerCAmelCase   =		model(__UpperCamelCase       , labels=__UpperCamelCase						)
											self.parent.assertEqual(result.logits.shape       , (self.batch_size, self.type_sequence_label_size)						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		self.prepare_config_and_inputs()
											lowerCAmelCase   ,     lowerCAmelCase   ,     lowerCAmelCase   =		config_and_inputs
											lowerCAmelCase   =		{'''pixel_values''': pixel_values}
											return config, inputs_dict




@require_torch
class 							__UpperCamelCase			( __UpperCAmelCase     ,					__UpperCAmelCase     ,					unittest.TestCase  ):





				'''simple docstring'''



				__a							: Optional[Any]										=(ViTHybridModel, ViTHybridForImageClassification) if is_torch_available() else ()
				__a							: Dict										=(
				    {"""feature-extraction""": ViTHybridModel, """image-classification""": ViTHybridForImageClassification}
				    if is_torch_available()
				    else {}
				)
				__a							: Optional[int]										=False
				__a							: Optional[int]										=False
				__a							: Optional[int]										=False



				def       __snake_case		( self						):
											lowerCAmelCase   =		ViTHybridModelTester(self						)
											lowerCAmelCase   =		ConfigTester(self       , config_class=__UpperCamelCase       , has_text_modality=__UpperCamelCase       , hidden_size=37						)



				def       __snake_case		( self						):
											self.config_tester.run_common_tests()



				@unittest.skip(reason='''ViT does not use inputs_embeds'''						)
				def       __snake_case		( self						):
											pass



				def       __snake_case		( self						):
											lowerCAmelCase   ,     lowerCAmelCase   =		self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		lowerCAmelCase   =		model_class(__UpperCamelCase						)
																		self.assertIsInstance(model.get_input_embeddings()       , (nn.Module)						)
																		lowerCAmelCase   =		model.get_output_embeddings()
																		self.assertTrue(x is None or isinstance(__UpperCamelCase       , nn.Linear						)						)



				def       __snake_case		( self						):
											lowerCAmelCase   ,     lowerCAmelCase   =		self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		lowerCAmelCase   =		model_class(__UpperCamelCase						)
																		lowerCAmelCase   =		inspect.signature(model.forward						)
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		lowerCAmelCase   =		[*signature.parameters.keys()]

																		lowerCAmelCase   =		['''pixel_values''']
																		self.assertListEqual(arg_names[:1]       , __UpperCamelCase						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__UpperCamelCase						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_image_classification(*__UpperCamelCase						)



				def       __snake_case		( self						):
											lowerCAmelCase   ,     lowerCAmelCase   =		self.model_tester.prepare_config_and_inputs_for_common()

											lowerCAmelCase   =		_config_zero_init(__UpperCamelCase						)
											for model_class in self.all_model_classes:
																		lowerCAmelCase   =		model_class(config=__UpperCamelCase						)
																		# Skip the check for the backbone
																		for name, module in model.named_modules():
																									if module.__class__.__name__ == "ViTHybridPatchEmbeddings":
																																lowerCAmelCase   =		[F"""{name}.{key}""" for key in module.state_dict().keys()]
																																break

																		for name, param in model.named_parameters():
																									if param.requires_grad:
																																if name in backbone_params:
																																							continue
																																self.assertIn(
																																    ((param.data.mean() * 1E9).round() / 1E9).item()       , [0.0, 1.0]       , msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""       , )






				@slow
				def       __snake_case		( self						):
											for model_name in VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		lowerCAmelCase   =		ViTHybridModel.from_pretrained(__UpperCamelCase						)
																		self.assertIsNotNone(__UpperCamelCase						)




def       UpperCAmelCase       (     ):
							lowerCAmelCase   =		Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
							return image

@require_torch
@require_vision
class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				@cached_property
				def       __snake_case		( self						):
											return (
											    ViTHybridImageProcessor.from_pretrained(VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST[0]						)
											    if is_vision_available()
											    else None
											)



				@slow
				def       __snake_case		( self						):
											lowerCAmelCase   =		ViTHybridForImageClassification.from_pretrained(VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST[0]						).to(
											    __UpperCamelCase						)

											lowerCAmelCase   =		self.default_image_processor
											lowerCAmelCase   =		prepare_img()
											lowerCAmelCase   =		image_processor(images=__UpperCamelCase       , return_tensors='''pt'''						).to(__UpperCamelCase						)

											# forward pass
											with torch.no_grad():
																		lowerCAmelCase   =		model(**__UpperCamelCase						)

											# verify the logits
											lowerCAmelCase   =		torch.Size((1, 10_00)						)
											self.assertEqual(outputs.logits.shape       , __UpperCamelCase						)

											lowerCAmelCase   =		torch.tensor([-1.9090, -0.4993, -0.2389]						).to(__UpperCamelCase						)

											self.assertTrue(torch.allclose(outputs.logits[0, :3]       , __UpperCamelCase       , atol=1E-4						)						)






				@slow
				@require_accelerate
				def       __snake_case		( self						):
											lowerCAmelCase   =		ViTHybridImageProcessor.from_pretrained('''google/vit-hybrid-base-bit-384'''						)
											lowerCAmelCase   =		ViTHybridForImageClassification.from_pretrained('''google/vit-hybrid-base-bit-384'''       , device_map='''auto'''						)

											lowerCAmelCase   =		prepare_img()

											lowerCAmelCase   =		image_processor(images=__UpperCamelCase       , return_tensors='''pt'''						)
											lowerCAmelCase   =		model(**__UpperCamelCase						)
											lowerCAmelCase   =		outputs.logits
											# model predicts one of the 1000 ImageNet classes
											lowerCAmelCase   =		logits.argmax(-1						).item()

											self.assertTrue(model.config.idalabel[predicted_class_idx]       , '''tabby, tabby cat'''						)









import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class 							__UpperCamelCase			( __UpperCAmelCase     ,					unittest.TestCase  ):





				'''simple docstring'''



				__a							: Any										=BertJapaneseTokenizer
				__a							: Optional[int]										=False
				__a							: int										=True



				def       __snake_case		( self						):
											super().setUp()

											lowerCAmelCase   =		[
											    '''[UNK]''',
											    '''[CLS]''',
											    '''[SEP]''',
											    '''こんにちは''',
											    '''こん''',
											    '''にちは''',
											    '''ばんは''',
											    '''##こん''',
											    '''##にちは''',
											    '''##ばんは''',
											    '''世界''',
											    '''##世界''',
											    '''、''',
											    '''##、''',
											    '''。''',
											    '''##。''',
											]

											lowerCAmelCase   =		os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['''vocab_file''']						)
											with open(self.vocab_file       , '''w'''       , encoding='''utf-8'''						) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		'''こんにちは、世界。 \nこんばんは、世界。'''
											lowerCAmelCase   =		'''こんにちは 、 世界 。 こんばんは 、 世界 。'''
											return input_text, output_text



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   ,     lowerCAmelCase   =		self.get_input_output_texts(UpperCAmelCase_						)
											lowerCAmelCase   =		tokenizer.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_						)
											lowerCAmelCase   =		tokenizer.decode(UpperCAmelCase_       , clean_up_tokenization_spaces=UpperCAmelCase_						)
											return text, ids



				def       __snake_case		( self						):
											pass  # TODO add if relevant



				def       __snake_case		( self						):
											pass  # TODO add if relevant



				def       __snake_case		( self						):
											pass  # TODO add if relevant



				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class(self.vocab_file						)

											lowerCAmelCase   =		tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''						)
											self.assertListEqual(UpperCAmelCase_       , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_						)       , [3, 12, 10, 14, 4, 9, 12, 10, 14]						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class(self.vocab_file       , word_tokenizer_type='''mecab'''						)
											self.assertIsNotNone(UpperCAmelCase_						)

											lowerCAmelCase   =		'''こんにちは、世界。\nこんばんは、世界。'''
											lowerCAmelCase   =		tokenizer.tokenize(UpperCAmelCase_						)
											self.assertListEqual(UpperCAmelCase_       , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_						)       , [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

											lowerCAmelCase   =		os.path.join(self.tmpdirname       , '''tokenizer.bin'''						)
											with open(UpperCAmelCase_       , '''wb'''						) as handle:
																		pickle.dump(UpperCAmelCase_       , UpperCAmelCase_						)

											with open(UpperCAmelCase_       , '''rb'''						) as handle:
																		lowerCAmelCase   =		pickle.load(UpperCAmelCase_						)

											lowerCAmelCase   =		tokenizer_new.tokenize(UpperCAmelCase_						)

											self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		MecabTokenizer(mecab_dic='''ipadic'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']       , )



				def       __snake_case		( self						):
											try:
																		lowerCAmelCase   =		MecabTokenizer(mecab_dic='''unidic_lite'''						)
											except ModuleNotFoundError:
																		return

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']       , )



				def       __snake_case		( self						):
											try:
																		lowerCAmelCase   =		MecabTokenizer(mecab_dic='''unidic'''						)
											except ModuleNotFoundError:
																		return

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']       , )



				def       __snake_case		( self						):
											lowerCAmelCase   =		MecabTokenizer(do_lower_case=UpperCAmelCase_       , mecab_dic='''ipadic'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']       , )



				def       __snake_case		( self						):
											try:
																		lowerCAmelCase   =		MecabTokenizer(
																		    do_lower_case=UpperCAmelCase_       , normalize_text=UpperCAmelCase_       , mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''						)
											except RuntimeError:
																		# if dict doesn't exist in the system, previous code raises this error.
																		return

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']       , )



				def       __snake_case		( self						):
											lowerCAmelCase   =		MecabTokenizer(normalize_text=UpperCAmelCase_       , mecab_dic='''ipadic'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']       , )



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class(self.vocab_file       , word_tokenizer_type='''sudachi'''						)
											self.assertIsNotNone(UpperCAmelCase_						)

											lowerCAmelCase   =		'''こんにちは、世界。\nこんばんは、世界。'''
											lowerCAmelCase   =		tokenizer.tokenize(UpperCAmelCase_						)
											self.assertListEqual(UpperCAmelCase_       , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_						)       , [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

											lowerCAmelCase   =		os.path.join(self.tmpdirname       , '''tokenizer.bin'''						)
											with open(UpperCAmelCase_       , '''wb'''						) as handle:
																		pickle.dump(UpperCAmelCase_       , UpperCAmelCase_						)

											with open(UpperCAmelCase_       , '''rb'''						) as handle:
																		lowerCAmelCase   =		pickle.load(UpperCAmelCase_						)

											lowerCAmelCase   =		tokenizer_new.tokenize(UpperCAmelCase_						)

											self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_						)



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(sudachi_dict_type='''core'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']       , )



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(sudachi_dict_type='''core'''       , sudachi_split_mode='''A'''						)

											self.assertListEqual(tokenizer.tokenize('''外国人参政権'''						)       , ['''外国''', '''人''', '''参政''', '''権''']						)



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(sudachi_dict_type='''core'''       , sudachi_split_mode='''B'''						)

											self.assertListEqual(tokenizer.tokenize('''外国人参政権'''						)       , ['''外国人''', '''参政権''']						)



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(sudachi_dict_type='''core'''       , sudachi_split_mode='''C'''						)

											self.assertListEqual(tokenizer.tokenize('''外国人参政権'''						)       , ['''外国人参政権''']						)



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(do_lower_case=UpperCAmelCase_       , sudachi_dict_type='''core'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']       , )



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(normalize_text=UpperCAmelCase_       , sudachi_dict_type='''core'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']       , )



				@require_sudachi
				def       __snake_case		( self						):
											lowerCAmelCase   =		SudachiTokenizer(trim_whitespace=UpperCAmelCase_       , sudachi_dict_type='''core'''						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']       , )



				@require_jumanpp
				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class(self.vocab_file       , word_tokenizer_type='''jumanpp'''						)
											self.assertIsNotNone(UpperCAmelCase_						)

											lowerCAmelCase   =		'''こんにちは、世界。\nこんばんは、世界。'''
											lowerCAmelCase   =		tokenizer.tokenize(UpperCAmelCase_						)
											self.assertListEqual(UpperCAmelCase_       , ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_						)       , [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

											lowerCAmelCase   =		os.path.join(self.tmpdirname       , '''tokenizer.bin'''						)
											with open(UpperCAmelCase_       , '''wb'''						) as handle:
																		pickle.dump(UpperCAmelCase_       , UpperCAmelCase_						)

											with open(UpperCAmelCase_       , '''rb'''						) as handle:
																		lowerCAmelCase   =		pickle.load(UpperCAmelCase_						)

											lowerCAmelCase   =		tokenizer_new.tokenize(UpperCAmelCase_						)

											self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_						)



				@require_jumanpp
				def       __snake_case		( self						):
											lowerCAmelCase   =		JumanppTokenizer()

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']       , )



				@require_jumanpp
				def       __snake_case		( self						):
											lowerCAmelCase   =		JumanppTokenizer(do_lower_case=UpperCAmelCase_						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']       , )



				@require_jumanpp
				def       __snake_case		( self						):
											lowerCAmelCase   =		JumanppTokenizer(normalize_text=UpperCAmelCase_						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']       , )



				@require_jumanpp
				def       __snake_case		( self						):
											lowerCAmelCase   =		JumanppTokenizer(trim_whitespace=UpperCAmelCase_						)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						)       , ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']       , )



				@require_jumanpp
				def       __snake_case		( self						):
											lowerCAmelCase   =		JumanppTokenizer()

											self.assertListEqual(
											    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''						)       , ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']       , )



				def       __snake_case		( self						):
											lowerCAmelCase   =		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

											lowerCAmelCase   =		{}
											for i, token in enumerate(UpperCAmelCase_						):
																		lowerCAmelCase   =		i
											lowerCAmelCase   =		WordpieceTokenizer(vocab=UpperCAmelCase_       , unk_token='''[UNK]'''						)

											self.assertListEqual(tokenizer.tokenize(''''''						)       , []						)

											self.assertListEqual(tokenizer.tokenize('''こんにちは'''						)       , ['''こんにちは''']						)

											self.assertListEqual(tokenizer.tokenize('''こんばんは'''						)       , ['''こん''', '''##ばんは''']						)

											self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''						)       , ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''						)
											lowerCAmelCase   =		tokenizer.subword_tokenizer

											lowerCAmelCase   =		subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''						)
											self.assertListEqual(UpperCAmelCase_       , ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']						)

											lowerCAmelCase   =		subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''						)
											self.assertListEqual(UpperCAmelCase_       , ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''						)

											lowerCAmelCase   =		tokenizer.encode('''ありがとう。'''       , add_special_tokens=UpperCAmelCase_						)
											lowerCAmelCase   =		tokenizer.encode('''どういたしまして。'''       , add_special_tokens=UpperCAmelCase_						)

											lowerCAmelCase   =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_						)
											lowerCAmelCase   =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_       , UpperCAmelCase_						)

											# 2 is for "[CLS]", 3 is for "[SEP]"
											assert encoded_sentence == [2] + text + [3]
											assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 							__UpperCamelCase			( __UpperCAmelCase     ,					unittest.TestCase  ):





				'''simple docstring'''



				__a							: Union[str, Any]										=BertJapaneseTokenizer
				__a							: Optional[int]										=False



				def       __snake_case		( self						):
											super().setUp()

											lowerCAmelCase   =		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

											lowerCAmelCase   =		os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES['''vocab_file''']						)
											with open(self.vocab_file       , '''w'''       , encoding='''utf-8'''						) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)



				def       __snake_case		( self       , **UpperCAmelCase_						):
											return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       , subword_tokenizer_type='''character'''       , **UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		'''こんにちは、世界。 \nこんばんは、世界。'''
											lowerCAmelCase   =		'''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
											return input_text, output_text



				def       __snake_case		( self						):
											pass  # TODO add if relevant



				def       __snake_case		( self						):
											pass  # TODO add if relevant



				def       __snake_case		( self						):
											pass  # TODO add if relevant



				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class(self.vocab_file       , subword_tokenizer_type='''character'''						)

											lowerCAmelCase   =		tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''						)
											self.assertListEqual(
											    UpperCAmelCase_       , ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']						)
											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(UpperCAmelCase_						)       , [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)



				def       __snake_case		( self						):
											lowerCAmelCase   =		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

											lowerCAmelCase   =		{}
											for i, token in enumerate(UpperCAmelCase_						):
																		lowerCAmelCase   =		i
											lowerCAmelCase   =		CharacterTokenizer(vocab=UpperCAmelCase_       , unk_token='''[UNK]'''						)

											self.assertListEqual(tokenizer.tokenize(''''''						)       , []						)

											self.assertListEqual(tokenizer.tokenize('''こんにちは'''						)       , ['''こ''', '''ん''', '''に''', '''ち''', '''は''']						)

											self.assertListEqual(tokenizer.tokenize('''こんにちほ'''						)       , ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''						)

											lowerCAmelCase   =		tokenizer.encode('''ありがとう。'''       , add_special_tokens=UpperCAmelCase_						)
											lowerCAmelCase   =		tokenizer.encode('''どういたしまして。'''       , add_special_tokens=UpperCAmelCase_						)

											lowerCAmelCase   =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_						)
											lowerCAmelCase   =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_       , UpperCAmelCase_						)

											# 2 is for "[CLS]", 3 is for "[SEP]"
											assert encoded_sentence == [2] + text + [3]
											assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self						):
											lowerCAmelCase   =		'''cl-tohoku/bert-base-japanese'''
											lowerCAmelCase   =		AutoTokenizer.from_pretrained(UpperCAmelCase_						)
											self.assertIsInstance(UpperCAmelCase_       , UpperCAmelCase_						)




class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self						):
											lowerCAmelCase   =		'''cl-tohoku/bert-base-japanese'''
											with self.assertLogs('''transformers'''       , level='''WARNING'''						) as cm:
																		BertTokenizer.from_pretrained(UpperCAmelCase_						)
																		self.assertTrue(
																		    cm.records[0].message.startswith(
																		        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																		        ''' is called from.'''						)						)
											lowerCAmelCase   =		'''bert-base-cased'''
											with self.assertLogs('''transformers'''       , level='''WARNING'''						) as cm:
																		BertJapaneseTokenizer.from_pretrained(UpperCAmelCase_						)
																		self.assertTrue(
																		    cm.records[0].message.startswith(
																		        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
																		        ''' is called from.'''						)						)



