



'''simple docstring'''



import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase	:		Any				=   get_tests_dir('''fixtures/test_sentencepiece.model''')






@require_sentencepiece
class 					UpperCAmelCase__	(		UpperCamelCase__							,       unittest.TestCase ):
					a      :     str        =							XLMProphetNetTokenizer
					a      :     Tuple        =							False
					a      :     Any        =							True



					def        UpperCAmelCase_ (    self     )     ->			Union[str, Any]:
						super().setUp()

						# We have a SentencePiece fixture for testing
						__lowerCAmelCase												=  XLMProphetNetTokenizer(UpperCamelCase			,				keep_accents=UpperCamelCase     )
						tokenizer.save_pretrained(self.tmpdirname     )



					def        UpperCAmelCase_ (    self     )     ->			Tuple:
						__lowerCAmelCase												=  "[PAD]"
						__lowerCAmelCase												=  0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase     )			,				UpperCamelCase     )



					def        UpperCAmelCase_ (    self     )     ->			List[str]:
						__lowerCAmelCase												=  list(self.get_tokenizer().get_vocab().keys()     )

						self.assertEqual(vocab_keys[0]			,				"[PAD]"     )
						self.assertEqual(vocab_keys[1]			,				"[CLS]"     )
						self.assertEqual(vocab_keys[-1]			,				"j"     )
						self.assertEqual(len(UpperCamelCase     )			,				1012     )



					def        UpperCAmelCase_ (    self     )     ->			Any:
						self.assertEqual(self.get_tokenizer().vocab_size			,				1012     )



					def        UpperCAmelCase_ (    self     )     ->			Dict:
						__lowerCAmelCase												=  XLMProphetNetTokenizer(UpperCamelCase			,				keep_accents=UpperCamelCase     )

						__lowerCAmelCase												=  tokenizer.tokenize("This is a test"     )
						self.assertListEqual(UpperCamelCase			,				["▁This", "▁is", "▁a", "▁t", "est"]     )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(UpperCamelCase     )			,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,				)

						__lowerCAmelCase												=  tokenizer.tokenize("I was born in 92000, and this is falsé."     )
						self.assertListEqual(
						    UpperCamelCase			,				[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "9",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "é",
						        ".",
						    ]			,				)
						__lowerCAmelCase												=  tokenizer.convert_tokens_to_ids(UpperCamelCase     )
						self.assertListEqual(
						    UpperCamelCase			,				[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]			,				)

						__lowerCAmelCase												=  tokenizer.convert_ids_to_tokens(UpperCamelCase     )
						self.assertListEqual(
						    UpperCamelCase			,				[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "[UNK]",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "[UNK]",
						        ".",
						    ]			,				)



					@cached_property
					def        UpperCAmelCase_ (    self     )     ->			List[str]:
						return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"     )



					@slow
					def        UpperCAmelCase_ (    self     )     ->			Optional[int]:
						__lowerCAmelCase												=  "Hello World!"
						__lowerCAmelCase												=  [3_5389, 6672, 49, 2]
						self.assertListEqual(UpperCamelCase			,				self.big_tokenizer.encode(UpperCamelCase     )     )



					@slow
					def        UpperCAmelCase_ (    self     )     ->			List[Any]:
						# fmt: off
						__lowerCAmelCase												=  {"input_ids": [[1_1073, 8_2783, 18, 26, 8_2783, 549, 5_1540, 248, 1_7209, 1301, 217, 20, 21_5186, 1325, 147, 1_7209, 1301, 217, 20, 5_6370, 53, 12_2020, 20, 1_6477, 27, 8_7355, 4548, 20, 4728, 7_8392, 17, 15_9969, 18, 26, 2_4491, 629, 15, 538, 2_2704, 5439, 15, 2788, 2_4491, 9885, 15, 4_3534, 605, 15, 814, 1_8403, 3_3200, 29, 15, 4_3534, 2_4458, 1_2410, 111, 2_4966, 8_3669, 9637, 14_4068, 26, 850, 2_2346, 27, 147, 2_4966, 8_3669, 8_3490, 26, 3_9113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 12_2020, 11_5785, 34, 816, 1339, 4_6887, 18, 147, 5_3905, 1951, 4_2238, 4_1170, 1_7732, 834, 436, 15, 2_7523, 9_8733, 217, 147, 5542, 4981, 930, 1_7347, 16, 2], [2_0091, 629, 94, 8_2786, 58, 490, 20, 1528, 84, 5_3905, 344, 8_0592, 11_0128, 1_8822, 5267, 1306, 62, 15_2537, 308, 7997, 401, 12_4427, 549, 3_5442, 225, 109, 1_5055, 2_5748, 147, 7119, 4_3712, 34, 767, 13_5366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 6_3784, 11_9466, 17, 14_7808, 8_8214, 18, 656, 81, 32, 3296, 1_0280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=UpperCamelCase			,				model_name="microsoft/xprophetnet-large-wiki100-cased"			,				revision="1acad1643ddd54a44df6a1b797ada8373685d90e"			,				)




'''simple docstring'''



import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling



def   __lowerCAmelCase					(							lowerCamelCase   :						Any					):






	'''simple docstring'''




	__lowerCAmelCase												=  [2, 2, 6, 2] if "tiny" in model_name else [2, 2, 18, 2]
	__lowerCAmelCase												=  True if "large" in model_name or "huge" in model_name else False
	__lowerCAmelCase												=  True if "large" in model_name or "huge" in model_name else False
	__lowerCAmelCase												=  True if "large" in model_name or "huge" in model_name else False

	if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
		if "fl3" in model_name:
			__lowerCAmelCase												=  [3, 3, 3, 3]
			__lowerCAmelCase												=  [5, 5, 5, 5]
		elif "fl4" in model_name:
			__lowerCAmelCase												=  [4, 4, 4, 4]
			__lowerCAmelCase												=  [3, 3, 3, 3]

	if "tiny" in model_name or "small" in model_name or "base" in model_name:
		__lowerCAmelCase												=  [3, 3, 3, 3]
		if "lrf" in model_name:
			__lowerCAmelCase												=  [3, 3, 3, 3]
		else:
			__lowerCAmelCase												=  [2, 2, 2, 2]

	if "tiny" in model_name:
		__lowerCAmelCase												=  96
	elif "small" in model_name:
		__lowerCAmelCase												=  96
	elif "base" in model_name:
		__lowerCAmelCase												=  1_28
	elif "large" in model_name:
		__lowerCAmelCase												=  1_92
	elif "xlarge" in model_name:
		__lowerCAmelCase												=  2_56
	elif "huge" in model_name:
		__lowerCAmelCase												=  3_52

	# set label information
	__lowerCAmelCase												=  "huggingface/label-files"
	if "large" in model_name or "huge" in model_name:
		__lowerCAmelCase												=  "imagenet-22k-id2label.json"
	else:
		__lowerCAmelCase												=  "imagenet-1k-id2label.json"

	__lowerCAmelCase												=  json.load(open(hf_hub_download(lowerCamelCase				,		lowerCamelCase				,		repo_type="dataset"					)				,		"r"					)					)
	__lowerCAmelCase												=  {int(lowerCamelCase					): v for k, v in idalabel.items()}
	__lowerCAmelCase												=  {v: k for k, v in idalabel.items()}

	__lowerCAmelCase												=  FocalNetConfig(
	    embed_dim=lowerCamelCase				,		depths=lowerCamelCase				,		focal_levels=lowerCamelCase				,		focal_windows=lowerCamelCase				,		use_conv_embed=lowerCamelCase				,		idalabel=lowerCamelCase				,		labelaid=lowerCamelCase				,		use_post_layernorm=lowerCamelCase				,		use_layerscale=lowerCamelCase				,		)

	return config



def   __lowerCAmelCase					(							lowerCamelCase   :						Union[str, Any]					):






	'''simple docstring'''




	if "patch_embed.proj" in name:
		__lowerCAmelCase												=  name.replace("patch_embed.proj"				,		"embeddings.patch_embeddings.projection"					)
	if "patch_embed.norm" in name:
		__lowerCAmelCase												=  name.replace("patch_embed.norm"				,		"embeddings.norm"					)
	if "layers" in name:
		__lowerCAmelCase												=  "encoder." + name
	if "encoder.layers" in name:
		__lowerCAmelCase												=  name.replace("encoder.layers"				,		"encoder.stages"					)
	if "downsample.proj" in name:
		__lowerCAmelCase												=  name.replace("downsample.proj"				,		"downsample.projection"					)
	if "blocks" in name:
		__lowerCAmelCase												=  name.replace("blocks"				,		"layers"					)
	if "modulation.f.weight" in name or "modulation.f.bias" in name:
		__lowerCAmelCase												=  name.replace("modulation.f"				,		"modulation.projection_in"					)
	if "modulation.h.weight" in name or "modulation.h.bias" in name:
		__lowerCAmelCase												=  name.replace("modulation.h"				,		"modulation.projection_context"					)
	if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
		__lowerCAmelCase												=  name.replace("modulation.proj"				,		"modulation.projection_out"					)

	if name == "norm.weight":
		__lowerCAmelCase												=  "layernorm.weight"
	if name == "norm.bias":
		__lowerCAmelCase												=  "layernorm.bias"

	if "head" in name:
		__lowerCAmelCase												=  name.replace("head"				,		"classifier"					)
	else:
		__lowerCAmelCase												=  "focalnet." + name

	return name



def   __lowerCAmelCase					(							lowerCamelCase   :						int				,		lowerCamelCase   :						Any				,		lowerCamelCase   :						Union[str, Any]=False					):






	'''simple docstring'''




	__lowerCAmelCase												=  {
	    "focalnet-tiny": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth",
	    "focalnet-tiny-lrf": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth",
	    "focalnet-small": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth",
	    "focalnet-small-lrf": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth",
	    "focalnet-base": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth",
	    "focalnet-base-lrf": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth",
	    "focalnet-large-lrf-fl3": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth",
	    "focalnet-large-lrf-fl4": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth",
	    "focalnet-xlarge-lrf-fl3": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth",
	    "focalnet-xlarge-lrf-fl4": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth",
	}
	# fmt: on

	__lowerCAmelCase												=  model_name_to_url[model_name]
	print("Checkpoint URL: "				,		lowerCamelCase					)
	__lowerCAmelCase												=  torch.hub.load_state_dict_from_url(lowerCamelCase				,		map_location="cpu"					)["model"]

	# rename keys
	for key in state_dict.copy().keys():
		__lowerCAmelCase												=  state_dict.pop(lowerCamelCase					)
		__lowerCAmelCase												=  val

	__lowerCAmelCase												=  get_focalnet_config(lowerCamelCase					)
	__lowerCAmelCase												=  FocalNetForImageClassification(lowerCamelCase					)
	model.eval()

	# load state dict
	model.load_state_dict(lowerCamelCase					)

	# verify conversion
	__lowerCAmelCase												=  "http://images.cocodataset.org/val2017/000000039769.jpg"

	__lowerCAmelCase												=  BitImageProcessor(
	    do_resize=lowerCamelCase				,		size={"shortest_edge": 2_56}				,		resample=PILImageResampling.BILINEAR				,		do_center_crop=lowerCamelCase				,		crop_size=2_24				,		do_normalize=lowerCamelCase				,		image_mean=lowerCamelCase				,		image_std=lowerCamelCase				,		)
	__lowerCAmelCase												=  Image.open(requests.get(lowerCamelCase				,		stream=lowerCamelCase					).raw					)
	__lowerCAmelCase												=  processor(images=lowerCamelCase				,		return_tensors="pt"					)

	__lowerCAmelCase												=  transforms.Compose(
	    [
	        transforms.Resize(2_56					),
	        transforms.CenterCrop(2_24					),
	        transforms.ToTensor(),
	        transforms.Normalize(mean=[0.4_8_5, 0.4_5_6, 0.4_0_6]				,		std=[0.2_2_9, 0.2_2_4, 0.2_2_5]					),
	    ]					)

	__lowerCAmelCase												=  image_transforms(lowerCamelCase					).unsqueeze(0					)

	# verify pixel_values
	assert torch.allclose(inputs.pixel_values				,		lowerCamelCase				,		atol=1e-4					)

	__lowerCAmelCase												=  model(**lowerCamelCase					)

	__lowerCAmelCase												=  outputs.logits.argmax(-1					).item()
	print("Predicted class:"				,		model.config.idalabel[predicted_class_idx]					)

	print("First values of logits:"				,		outputs.logits[0, :3]					)

	if model_name == "focalnet-tiny":
		__lowerCAmelCase												=  torch.tensor([0.2_1_6_6, -0.4_3_6_8, 0.2_1_9_1]					)
	elif model_name == "focalnet-tiny-lrf":
		__lowerCAmelCase												=  torch.tensor([1.1_6_6_9, 0.0_1_2_5, -0.1_6_9_5]					)
	elif model_name == "focalnet-small":
		__lowerCAmelCase												=  torch.tensor([0.4_9_1_7, -0.0_4_3_0, 0.1_3_4_1]					)
	elif model_name == "focalnet-small-lrf":
		__lowerCAmelCase												=  torch.tensor([-0.2_5_8_8, -0.5_3_4_2, -0.2_3_3_1]					)
	elif model_name == "focalnet-base":
		__lowerCAmelCase												=  torch.tensor([-0.1_6_5_5, -0.4_0_9_0, -0.1_7_3_0]					)
	elif model_name == "focalnet-base-lrf":
		__lowerCAmelCase												=  torch.tensor([0.5_3_0_6, -0.0_4_8_3, -0.3_9_2_8]					)
	assert torch.allclose(outputs.logits[0, :3]				,		lowerCamelCase				,		atol=1e-4					)
	print("Looks ok!"					)

	if pytorch_dump_folder_path is not None:
		print(f'''Saving model and processor of {model_name} to {pytorch_dump_folder_path}'''					)
		model.save_pretrained(lowerCamelCase					)
		processor.save_pretrained(lowerCamelCase					)

	if push_to_hub:
		print(f'''Pushing model and processor of {model_name} to the hub...'''					)
		model.push_to_hub(f'''{model_name}'''					)
		processor.push_to_hub(f'''{model_name}'''					)


if __name__ == "__main__":
	lowerCAmelCase	:		Tuple				=   argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--model_name''',
	    default='''focalnet-tiny''',
	    type=str,
	    help='''Name of the FocalNet model you\'d like to convert.''',
	)
	parser.add_argument(
	    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
	)
	parser.add_argument(
	    '''--push_to_hub''',
	    action='''store_true''',
	    help='''Whether to push the model and processor to the hub.''',
	)

	lowerCAmelCase	:		Optional[int]				=   parser.parse_args()
	convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)