





import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							List[Any]=False    ):
		SCREAMING_SNAKE_CASE__			=      []
		for i in range(config.num_hidden_layers    ):
				# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
				rename_keys.append((f'''module.blocks.{i}.norm1.weight''', f'''vit.encoder.layer.{i}.layernorm_before.weight''')    )
				rename_keys.append((f'''module.blocks.{i}.norm1.bias''', f'''vit.encoder.layer.{i}.layernorm_before.bias''')    )
				rename_keys.append(
				    (f'''module.blocks.{i}.attn.proj.weight''', f'''vit.encoder.layer.{i}.attention.output.dense.weight''')    )
				rename_keys.append((f'''module.blocks.{i}.attn.proj.bias''', f'''vit.encoder.layer.{i}.attention.output.dense.bias''')    )
				rename_keys.append((f'''module.blocks.{i}.norm2.weight''', f'''vit.encoder.layer.{i}.layernorm_after.weight''')    )
				rename_keys.append((f'''module.blocks.{i}.norm2.bias''', f'''vit.encoder.layer.{i}.layernorm_after.bias''')    )
				rename_keys.append((f'''module.blocks.{i}.mlp.fc1.weight''', f'''vit.encoder.layer.{i}.intermediate.dense.weight''')    )
				rename_keys.append((f'''module.blocks.{i}.mlp.fc1.bias''', f'''vit.encoder.layer.{i}.intermediate.dense.bias''')    )
				rename_keys.append((f'''module.blocks.{i}.mlp.fc2.weight''', f'''vit.encoder.layer.{i}.output.dense.weight''')    )
				rename_keys.append((f'''module.blocks.{i}.mlp.fc2.bias''', f'''vit.encoder.layer.{i}.output.dense.bias''')    )

		# projection layer + position embeddings
		rename_keys.extend(
		    [
		        ("""module.cls_token""", """vit.embeddings.cls_token"""),
		        ("""module.patch_embed.proj.weight""", """vit.embeddings.patch_embeddings.projection.weight"""),
		        ("""module.patch_embed.proj.bias""", """vit.embeddings.patch_embeddings.projection.bias"""),
		        ("""module.pos_embed""", """vit.embeddings.position_embeddings"""),
		    ]    )

		if base_model:
				# layernorm + pooler
				rename_keys.extend(
				    [
				        ("""module.norm.weight""", """layernorm.weight"""),
				        ("""module.norm.bias""", """layernorm.bias"""),
				    ]    )

				# if just the base model, we should remove "vit" from all keys that start with "vit"
				SCREAMING_SNAKE_CASE__			=      [(pair[0], pair[1][4:]) if pair[1].startswith("""vit"""    ) else pair for pair in rename_keys]
		else:
				# layernorm + classification head
				rename_keys.extend(
				    [
				        ("""norm.weight""", """vit.layernorm.weight"""),
				        ("""norm.bias""", """vit.layernorm.bias"""),
				        ("""head.weight""", """classifier.weight"""),
				        ("""head.bias""", """classifier.bias"""),
				    ]    )

		return rename_keys






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Optional[Any]=False    ):
		for i in range(config.num_hidden_layers    ):
				if base_model:
						SCREAMING_SNAKE_CASE__			=      """"""
				else:
						SCREAMING_SNAKE_CASE__			=      """vit."""
				# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
				SCREAMING_SNAKE_CASE__			=      state_dict.pop(f'''module.blocks.{i}.attn.qkv.weight'''    )
				SCREAMING_SNAKE_CASE__			=      state_dict.pop(f'''module.blocks.{i}.attn.qkv.bias'''    )
				# next, add query, keys and values (in that order) to the state dict
				SCREAMING_SNAKE_CASE__			=      in_proj_weight[
				    : config.hidden_size, :
				]
				SCREAMING_SNAKE_CASE__			=      in_proj_bias[: config.hidden_size]
				SCREAMING_SNAKE_CASE__			=      in_proj_weight[
				    config.hidden_size : config.hidden_size * 2, :
				]
				SCREAMING_SNAKE_CASE__			=      in_proj_bias[
				    config.hidden_size : config.hidden_size * 2
				]
				SCREAMING_SNAKE_CASE__			=      in_proj_weight[
				    -config.hidden_size :, :
				]
				SCREAMING_SNAKE_CASE__			=      in_proj_bias[-config.hidden_size :]






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict    ):
		SCREAMING_SNAKE_CASE__			=      ["""head.weight""", """head.bias"""]
		for k in ignore_keys:
				state_dict.pop(UpperCamelCase__					,   UpperCamelCase__    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict    ):
		# projection head is used in the self-supervised pre-training in MSN,
		# for downstream task it's not needed.
		SCREAMING_SNAKE_CASE__			=      [
		    """module.fc.fc1.weight""",
		    """module.fc.fc1.bias""",
		    """module.fc.bn1.weight""",
		    """module.fc.bn1.bias""",
		    """module.fc.bn1.running_mean""",
		    """module.fc.bn1.running_var""",
		    """module.fc.bn1.num_batches_tracked""",
		    """module.fc.fc2.weight""",
		    """module.fc.fc2.bias""",
		    """module.fc.bn2.weight""",
		    """module.fc.bn2.bias""",
		    """module.fc.bn2.running_mean""",
		    """module.fc.bn2.running_var""",
		    """module.fc.bn2.num_batches_tracked""",
		    """module.fc.fc3.weight""",
		    """module.fc.fc3.bias""",
		]
		for k in ignore_keys:
				state_dict.pop(UpperCamelCase__					,   UpperCamelCase__    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							int					,   UpperCamelCase__:							str					,   UpperCamelCase__:							Optional[int]    ):
		SCREAMING_SNAKE_CASE__			=      dct.pop(UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      val






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[Any]					,   UpperCamelCase__:							int    ):
		SCREAMING_SNAKE_CASE__			=      ViTMSNConfig()
		SCREAMING_SNAKE_CASE__			=      1_000

		SCREAMING_SNAKE_CASE__			=      """datasets/huggingface/label-files"""
		SCREAMING_SNAKE_CASE__			=      """imagenet-1k-id2label.json"""
		SCREAMING_SNAKE_CASE__			=      json.load(open(hf_hub_download(UpperCamelCase__					,   UpperCamelCase__    )					,   """r"""    )    )
		SCREAMING_SNAKE_CASE__			=      {int(UpperCamelCase__    ): v for k, v in idalabel.items()}
		SCREAMING_SNAKE_CASE__			=      idalabel
		SCREAMING_SNAKE_CASE__			=      {v: k for k, v in idalabel.items()}

		if "s16" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      384
				SCREAMING_SNAKE_CASE__			=      1_536
				SCREAMING_SNAKE_CASE__			=      6
		elif "l16" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      1_024
				SCREAMING_SNAKE_CASE__			=      4_096
				SCREAMING_SNAKE_CASE__			=      24
				SCREAMING_SNAKE_CASE__			=      16
				SCREAMING_SNAKE_CASE__			=      0.1
		elif "b4" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      4
		elif "l7" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      7
				SCREAMING_SNAKE_CASE__			=      1_024
				SCREAMING_SNAKE_CASE__			=      4_096
				SCREAMING_SNAKE_CASE__			=      24
				SCREAMING_SNAKE_CASE__			=      16
				SCREAMING_SNAKE_CASE__			=      0.1

		SCREAMING_SNAKE_CASE__			=      ViTMSNModel(UpperCamelCase__    )

		SCREAMING_SNAKE_CASE__			=      torch.hub.load_state_dict_from_url(UpperCamelCase__					,   map_location="""cpu"""    )["""target_encoder"""]

		SCREAMING_SNAKE_CASE__			=      ViTImageProcessor(size=config.image_size    )

		remove_projection_head(UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      create_rename_keys(UpperCamelCase__					,   base_model=UpperCamelCase__    )

		for src, dest in rename_keys:
				rename_key(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
		read_in_q_k_v(UpperCamelCase__					,   UpperCamelCase__					,   base_model=UpperCamelCase__    )

		model.load_state_dict(UpperCamelCase__    )
		model.eval()

		SCREAMING_SNAKE_CASE__			=      """http://images.cocodataset.org/val2017/000000039769.jpg"""

		SCREAMING_SNAKE_CASE__			=      Image.open(requests.get(UpperCamelCase__					,   stream=UpperCamelCase__    ).raw    )
		SCREAMING_SNAKE_CASE__			=      ViTImageProcessor(
		    size=config.image_size					,   image_mean=UpperCamelCase__					,   image_std=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      image_processor(images=UpperCamelCase__					,   return_tensors="""pt"""    )

		# forward pass
		torch.manual_seed(2    )
		SCREAMING_SNAKE_CASE__			=      model(**UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      outputs.last_hidden_state

		# The following Colab Notebook was used to generate these outputs:
		# https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
		if "s16" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      torch.tensor([[-1.0_9_1_5, -1.4_8_7_6, -1.1_8_0_9]]    )
		elif "b16" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      torch.tensor([[1_4.2_8_8_9, -1_8.9_0_4_5, 1_1.7_2_8_1]]    )
		elif "l16" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      torch.tensor([[4_1.5_0_2_8, -2_2.8_6_8_1, 4_5.6_4_7_5]]    )
		elif "b4" in checkpoint_url:
				SCREAMING_SNAKE_CASE__			=      torch.tensor([[-4.3_8_6_8, 5.2_9_3_2, -0.4_1_3_7]]    )
		else:
				SCREAMING_SNAKE_CASE__			=      torch.tensor([[-0.1_7_9_2, -0.6_4_6_5, 2.4_2_6_3]]    )

		# verify logits
		assert torch.allclose(last_hidden_state[:, 0, :3]					,   UpperCamelCase__					,   atol=1e-4    )

		print(f'''Saving model to {pytorch_dump_folder_path}'''    )
		model.save_pretrained(UpperCamelCase__    )

		print(f'''Saving image processor to {pytorch_dump_folder_path}'''    )
		image_processor.save_pretrained(UpperCamelCase__    )


if __name__ == "__main__":
	_lowerCamelCase   				=			argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--checkpoint_url',
	    default='https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar',
	    type=str,
	    help='URL of the checkpoint you\'d like to convert.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
	)

	_lowerCamelCase   				=			parser.parse_args()
	convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					RoCBertTokenizer
							lowerCamelCase_           =					None
							lowerCamelCase_           =					False
							lowerCamelCase_           =					True
							lowerCamelCase_           =					filter_non_english


							def      _snake_case  (							self    :List[Any]		)   ->   List[Any]:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									for i, value in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_shape_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_pronunciation_file"""]		)
									with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
									with open(self.word_shape_file							,				"""w"""							,				encoding="""utf-8"""		) as word_shape_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)
									with open(self.word_pronunciation_file							,				"""w"""							,				encoding="""utf-8"""		) as word_pronunciation_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""你好[SEP]你是谁"""		)
									self.assertListEqual(__A							,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)


							def      _snake_case  (							self    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""		)							,				["""ah""", """\u535A""", """\u63A8""", """zz"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""hello""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""h\u00E9llo"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :int		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				never_split=["""[UNK]"""]		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									SCREAMING_SNAKE_CASE__			=      {}
									for i, token in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      RoCBertWordpieceTokenizer(vocab=__A							,				unk_token="""[UNK]"""		)

									self.assertListEqual(tokenizer.tokenize(""""""		)							,				[]		)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""		)							,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]		)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""		)							,				["""[UNK]""", """runn""", """##ing"""]		)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_whitespace(""" """		)		)
									self.assertTrue(_is_whitespace("""\t"""		)		)
									self.assertTrue(_is_whitespace("""\r"""		)		)
									self.assertTrue(_is_whitespace("""\n"""		)		)
									self.assertTrue(_is_whitespace("""\u00A0"""		)		)

									self.assertFalse(_is_whitespace("""A"""		)		)
									self.assertFalse(_is_whitespace("""-"""		)		)


							def      _snake_case  (							self    :int		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_control("""\u0005"""		)		)

									self.assertFalse(_is_control("""A"""		)		)
									self.assertFalse(_is_control(""" """		)		)
									self.assertFalse(_is_control("""\t"""		)		)
									self.assertFalse(_is_control("""\r"""		)		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									self.assertTrue(_is_punctuation("""-"""		)		)
									self.assertTrue(_is_punctuation("""$"""		)		)
									self.assertTrue(_is_punctuation("""`"""		)		)
									self.assertTrue(_is_punctuation("""."""		)		)

									self.assertFalse(_is_punctuation("""A"""		)		)
									self.assertFalse(_is_punctuation(""" """		)		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)

									if self.test_rust_tokenizer:
											SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)


							def      _snake_case  (							self    :int		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode_plus(
													    __A							,				return_attention_mask=__A							,				return_token_type_ids=__A							,				return_offsets_mapping=__A							,				add_special_tokens=__A							,				)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.do_lower_case if hasattr(__A							,				"""do_lower_case"""		) else False
													SCREAMING_SNAKE_CASE__			=      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """A"""),
													        ((1, 2), ""","""),
													        ((3, 5), """na"""),
													        ((5, 6), """##ï"""),
													        ((6, 8), """##ve"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """Allen"""),
													        ((21, 23), """##NL"""),
													        ((23, 24), """##P"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """a"""),
													        ((1, 2), ""","""),
													        ((3, 8), """naive"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """allen"""),
													        ((21, 23), """##nl"""),
													        ((23, 24), """##p"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]		)		)
													self.assertEqual([e[0] for e in expected_results]							,				tokens["""offset_mapping"""]		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""的""", """人""", """有"""]
									SCREAMING_SNAKE_CASE__			=      """""".join(__A		)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      True
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)

													SCREAMING_SNAKE_CASE__			=      False
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE__			=      [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__A		)
													]
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)


							@slow
							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你好"""							,				add_special_tokens=__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你是谁"""							,				add_special_tokens=__A		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A							,				__A		)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizers(do_lower_case=__A		)
									for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
													SCREAMING_SNAKE_CASE__			=      """你好，你是谁"""
													SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_shape_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_pronunciation_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.prepare_for_model(
													    __A							,				__A							,				__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer.encode_plus(__A							,				add_special_tokens=__A		)

													self.assertEqual(__A							,				__A		)