





import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)







def       UpperCAmelCase							(     _lowerCamelCase      :  int			,   _lowerCamelCase      :  Dict=False     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Tuple   =					[]
						for i in range(config.num_hidden_layers     ):
												# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
												rename_keys.append((f"""module.blocks.{i}.norm1.weight""", f"""vit.encoder.layer.{i}.layernorm_before.weight""")     )
												rename_keys.append((f"""module.blocks.{i}.norm1.bias""", f"""vit.encoder.layer.{i}.layernorm_before.bias""")     )
												rename_keys.append(
												    (f"""module.blocks.{i}.attn.proj.weight""", f"""vit.encoder.layer.{i}.attention.output.dense.weight""")     )
												rename_keys.append((f"""module.blocks.{i}.attn.proj.bias""", f"""vit.encoder.layer.{i}.attention.output.dense.bias""")     )
												rename_keys.append((f"""module.blocks.{i}.norm2.weight""", f"""vit.encoder.layer.{i}.layernorm_after.weight""")     )
												rename_keys.append((f"""module.blocks.{i}.norm2.bias""", f"""vit.encoder.layer.{i}.layernorm_after.bias""")     )
												rename_keys.append((f"""module.blocks.{i}.mlp.fc1.weight""", f"""vit.encoder.layer.{i}.intermediate.dense.weight""")     )
												rename_keys.append((f"""module.blocks.{i}.mlp.fc1.bias""", f"""vit.encoder.layer.{i}.intermediate.dense.bias""")     )
												rename_keys.append((f"""module.blocks.{i}.mlp.fc2.weight""", f"""vit.encoder.layer.{i}.output.dense.weight""")     )
												rename_keys.append((f"""module.blocks.{i}.mlp.fc2.bias""", f"""vit.encoder.layer.{i}.output.dense.bias""")     )

						# projection layer + position embeddings
						rename_keys.extend(
						    [
						        ("module.cls_token", "vit.embeddings.cls_token"),
						        ("module.patch_embed.proj.weight", "vit.embeddings.patch_embeddings.projection.weight"),
						        ("module.patch_embed.proj.bias", "vit.embeddings.patch_embeddings.projection.bias"),
						        ("module.pos_embed", "vit.embeddings.position_embeddings"),
						    ]     )

						if base_model:
												# layernorm + pooler
												rename_keys.extend(
												    [
												        ("module.norm.weight", "layernorm.weight"),
												        ("module.norm.bias", "layernorm.bias"),
												    ]     )

												# if just the base model, we should remove "vit" from all keys that start with "vit"
												SCREAMING_SNAKE_CASE__     :				Dict   =					[(pair[0], pair[1][4:]) if pair[1].startswith("vit"     ) else pair for pair in rename_keys]
						else:
												# layernorm + classification head
												rename_keys.extend(
												    [
												        ("norm.weight", "vit.layernorm.weight"),
												        ("norm.bias", "vit.layernorm.bias"),
												        ("head.weight", "classifier.weight"),
												        ("head.bias", "classifier.bias"),
												    ]     )

						return rename_keys







def       UpperCAmelCase							(     _lowerCamelCase      :  Union[str, Any]			,   _lowerCamelCase      :  int			,   _lowerCamelCase      :  List[str]=False     ):





						'''simple docstring'''


						for i in range(config.num_hidden_layers     ):
												if base_model:
																		SCREAMING_SNAKE_CASE__     :				Dict   =					""
												else:
																		SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					"vit."
												# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
												SCREAMING_SNAKE_CASE__     :				Any   =					state_dict.pop(f"""module.blocks.{i}.attn.qkv.weight"""     )
												SCREAMING_SNAKE_CASE__     :				List[str]   =					state_dict.pop(f"""module.blocks.{i}.attn.qkv.bias"""     )
												# next, add query, keys and values (in that order) to the state dict
												SCREAMING_SNAKE_CASE__     :				Tuple   =					in_proj_weight[
												    : config.hidden_size, :
												]
												SCREAMING_SNAKE_CASE__     :				str   =					in_proj_bias[: config.hidden_size]
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					in_proj_weight[
												    config.hidden_size : config.hidden_size * 2, :
												]
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					in_proj_bias[
												    config.hidden_size : config.hidden_size * 2
												]
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					in_proj_weight[
												    -config.hidden_size :, :
												]
												SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					in_proj_bias[-config.hidden_size :]







def       UpperCAmelCase							(     _lowerCamelCase      :  List[str]     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Dict   =					["head.weight", "head.bias"]
						for k in ignore_keys:
												state_dict.pop(_lowerCamelCase			,   _lowerCamelCase     )







def       UpperCAmelCase							(     _lowerCamelCase      :  str     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				List[Any]   =					[
						    "module.fc.fc1.weight",
						    "module.fc.fc1.bias",
						    "module.fc.bn1.weight",
						    "module.fc.bn1.bias",
						    "module.fc.bn1.running_mean",
						    "module.fc.bn1.running_var",
						    "module.fc.bn1.num_batches_tracked",
						    "module.fc.fc2.weight",
						    "module.fc.fc2.bias",
						    "module.fc.bn2.weight",
						    "module.fc.bn2.bias",
						    "module.fc.bn2.running_mean",
						    "module.fc.bn2.running_var",
						    "module.fc.bn2.num_batches_tracked",
						    "module.fc.fc3.weight",
						    "module.fc.fc3.bias",
						]
						for k in ignore_keys:
												state_dict.pop(_lowerCamelCase			,   _lowerCamelCase     )







def       UpperCAmelCase							(     _lowerCamelCase      :  Optional[int]			,   _lowerCamelCase      :  List[str]			,   _lowerCamelCase      :  int     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				List[Any]   =					dct.pop(_lowerCamelCase     )
						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					val







def       UpperCAmelCase							(     _lowerCamelCase      :  List[str]			,   _lowerCamelCase      :  Optional[int]     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				List[str]   =					ViTMSNConfig()
						SCREAMING_SNAKE_CASE__     :				Any   =					1_000

						SCREAMING_SNAKE_CASE__     :				Optional[int]   =					"datasets/huggingface/label-files"
						SCREAMING_SNAKE_CASE__     :				Any   =					"imagenet-1k-id2label.json"
						SCREAMING_SNAKE_CASE__     :				Any   =					json.load(open(hf_hub_download(_lowerCamelCase			,   _lowerCamelCase     )			,   "r"     )     )
						SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					{int(_lowerCamelCase     ): v for k, v in idalabel.items()}
						SCREAMING_SNAKE_CASE__     :				Dict   =					idalabel
						SCREAMING_SNAKE_CASE__     :				List[str]   =					{v: k for k, v in idalabel.items()}

						if "s16" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					384
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					1_536
												SCREAMING_SNAKE_CASE__     :				List[Any]   =					6
						elif "l16" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				str   =					1_024
												SCREAMING_SNAKE_CASE__     :				str   =					4_096
												SCREAMING_SNAKE_CASE__     :				Dict   =					24
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					16
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					0.1
						elif "b4" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					4
						elif "l7" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				int   =					7
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					1_024
												SCREAMING_SNAKE_CASE__     :				List[Any]   =					4_096
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					24
												SCREAMING_SNAKE_CASE__     :				Tuple   =					16
												SCREAMING_SNAKE_CASE__     :				int   =					0.1

						SCREAMING_SNAKE_CASE__     :				str   =					ViTMSNModel(_lowerCamelCase     )

						SCREAMING_SNAKE_CASE__     :				List[str]   =					torch.hub.load_state_dict_from_url(_lowerCamelCase			,   map_location="cpu"     )["target_encoder"]

						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					ViTImageProcessor(size=config.image_size     )

						remove_projection_head(_lowerCamelCase     )
						SCREAMING_SNAKE_CASE__     :				str   =					create_rename_keys(_lowerCamelCase			,   base_model=_lowerCamelCase     )

						for src, dest in rename_keys:
												rename_key(_lowerCamelCase			,   _lowerCamelCase			,   _lowerCamelCase     )
						read_in_q_k_v(_lowerCamelCase			,   _lowerCamelCase			,   base_model=_lowerCamelCase     )

						model.load_state_dict(_lowerCamelCase     )
						model.eval()

						SCREAMING_SNAKE_CASE__     :				Tuple   =					"http://images.cocodataset.org/val2017/000000039769.jpg"

						SCREAMING_SNAKE_CASE__     :				Tuple   =					Image.open(requests.get(_lowerCamelCase			,   stream=_lowerCamelCase     ).raw     )
						SCREAMING_SNAKE_CASE__     :				Dict   =					ViTImageProcessor(
						    size=config.image_size			,   image_mean=_lowerCamelCase			,   image_std=_lowerCamelCase     )
						SCREAMING_SNAKE_CASE__     :				str   =					image_processor(images=_lowerCamelCase			,   return_tensors="pt"     )

						# forward pass
						torch.manual_seed(2     )
						SCREAMING_SNAKE_CASE__     :				Tuple   =					model(**_lowerCamelCase     )
						SCREAMING_SNAKE_CASE__     :				Any   =					outputs.last_hidden_state

						# The following Colab Notebook was used to generate these outputs:
						# https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
						if "s16" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					torch.tensor([[-1.0_9_1_5, -1.4_8_7_6, -1.1_8_0_9]]     )
						elif "b16" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				Dict   =					torch.tensor([[1_4.2_8_8_9, -1_8.9_0_4_5, 1_1.7_2_8_1]]     )
						elif "l16" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					torch.tensor([[4_1.5_0_2_8, -2_2.8_6_8_1, 4_5.6_4_7_5]]     )
						elif "b4" in checkpoint_url:
												SCREAMING_SNAKE_CASE__     :				int   =					torch.tensor([[-4.3_8_6_8, 5.2_9_3_2, -0.4_1_3_7]]     )
						else:
												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					torch.tensor([[-0.1_7_9_2, -0.6_4_6_5, 2.4_2_6_3]]     )

						# verify logits
						assert torch.allclose(last_hidden_state[:, 0, :3]			,   _lowerCamelCase			,   atol=1E-4     )

						print(f"""Saving model to {pytorch_dump_folder_path}"""     )
						model.save_pretrained(_lowerCamelCase     )

						print(f"""Saving image processor to {pytorch_dump_folder_path}"""     )
						image_processor.save_pretrained(_lowerCamelCase     )


if __name__ == "__main__":
				__lowercase       :List[str]       =  argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    "--checkpoint_url",
				    default="https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar",
				    type=str,
				    help="URL of the checkpoint you'd like to convert.",
				)
				parser.add_argument(
				    "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
				)

				__lowercase       :Dict       =  parser.parse_args()
				convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)






import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


__lowercase       :List[str]       =  get_tests_dir("fixtures/dummy_feature_extractor_config.json")
__lowercase       :str       =  get_tests_dir("fixtures/vocab.json")
__lowercase       :Optional[int]       =  get_tests_dir("fixtures")



class    _a     (				unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

							def      A_  (      self				:     Optional[Any]    )	->int:
													SCREAMING_SNAKE_CASE__     :				Dict   =					0

							def      A_  (      self				:     Any    )	->Optional[int]:
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"    )
													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     Union[str, Any]    )	->List[str]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				Dict   =					WavaVecaConfig()
																			SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"    )

																			# save in new folder
																			model_config.save_pretrained(a    )
																			processor.save_pretrained(a    )

																			SCREAMING_SNAKE_CASE__     :				str   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     int    )	->List[str]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			# copy relevant files
																			copyfile(a							,	os.path.join(a							,	a    )    )
																			copyfile(a							,	os.path.join(a							,	"vocab.json"    )    )

																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     List[Any]    )	->Tuple:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					WavaVecaFeatureExtractor()
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"    )

																			SCREAMING_SNAKE_CASE__     :				Any   =					WavaVecaProcessor(a							,	a    )

																			# save in new folder
																			processor.save_pretrained(a    )

																			# drop `processor_class` in tokenizer
																			with open(os.path.join(a							,	a    )							,	"r"    ) as f:
																									SCREAMING_SNAKE_CASE__     :				Optional[int]   =					json.load(a    )
																									config_dict.pop("processor_class"    )

																			with open(os.path.join(a							,	a    )							,	"w"    ) as f:
																									f.write(json.dumps(a    )    )

																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     List[str]    )	->Optional[Any]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					WavaVecaFeatureExtractor()
																			SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"    )

																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					WavaVecaProcessor(a							,	a    )

																			# save in new folder
																			processor.save_pretrained(a    )

																			# drop `processor_class` in feature extractor
																			with open(os.path.join(a							,	a    )							,	"r"    ) as f:
																									SCREAMING_SNAKE_CASE__     :				List[Any]   =					json.load(a    )
																									config_dict.pop("processor_class"    )

																			with open(os.path.join(a							,	a    )							,	"w"    ) as f:
																									f.write(json.dumps(a    )    )

																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     Union[str, Any]    )	->str:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					WavaVecaConfig(processor_class="Wav2Vec2Processor"    )
																			model_config.save_pretrained(a    )
																			# copy relevant files
																			copyfile(a							,	os.path.join(a							,	"vocab.json"    )    )
																			# create emtpy sample processor
																			with open(os.path.join(a							,	a    )							,	"w"    ) as f:
																									f.write("{}"    )

																			SCREAMING_SNAKE_CASE__     :				Tuple   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     Optional[Any]    )	->Optional[int]:
													# If remote code is not set, we will time out when asking whether to load the model.
													with self.assertRaises(a    ):
																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"    )
													# If remote code is disabled, we can't load this config.
													with self.assertRaises(a    ):
																			SCREAMING_SNAKE_CASE__     :				Any   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )

													SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )
													self.assertTrue(processor.special_attribute_present    )
													self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )

													SCREAMING_SNAKE_CASE__     :				Dict   =					processor.feature_extractor
													self.assertTrue(feature_extractor.special_attribute_present    )
													self.assertEqual(feature_extractor.__class__.__name__							,	"NewFeatureExtractor"    )

													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					processor.tokenizer
													self.assertTrue(tokenizer.special_attribute_present    )
													if is_tokenizers_available():
																			self.assertEqual(tokenizer.__class__.__name__							,	"NewTokenizerFast"    )

																			# Test we can also load the slow version
																			SCREAMING_SNAKE_CASE__     :				int   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a							,	use_fast=a    )
																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					new_processor.tokenizer
																			self.assertTrue(new_tokenizer.special_attribute_present    )
																			self.assertEqual(new_tokenizer.__class__.__name__							,	"NewTokenizer"    )
													else:
																			self.assertEqual(tokenizer.__class__.__name__							,	"NewTokenizer"    )

							def      A_  (      self				:     Tuple    )	->List[Any]:
													try:
																			AutoConfig.register("custom"							,	a    )
																			AutoFeatureExtractor.register(a							,	a    )
																			AutoTokenizer.register(a							,	slow_tokenizer_class=a    )
																			AutoProcessor.register(a							,	a    )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(a    ):
																									AutoProcessor.register(a							,	a    )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			SCREAMING_SNAKE_CASE__     :				List[str]   =					CustomFeatureExtractor.from_pretrained(a    )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																									SCREAMING_SNAKE_CASE__     :				int   =					os.path.join(a							,	"vocab.txt"    )
																									with open(a							,	"w"							,	encoding="utf-8"    ) as vocab_writer:
																															vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
																									SCREAMING_SNAKE_CASE__     :				Optional[int]   =					CustomTokenizer(a    )

																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					CustomProcessor(a							,	a    )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																									processor.save_pretrained(a    )
																									SCREAMING_SNAKE_CASE__     :				Any   =					AutoProcessor.from_pretrained(a    )
																									self.assertIsInstance(a							,	a    )

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in PROCESSOR_MAPPING._extra_content:
																									del PROCESSOR_MAPPING._extra_content[CustomConfig]

							def      A_  (      self				:     Union[str, Any]    )	->int:



													class    _a     (				lowercase__							):






																				"""simple docstring"""

																				snake_case_							 =						False







													class    _a     (				lowercase__							):






																				"""simple docstring"""

																				snake_case_							 =						False







													class    _a     (				lowercase__							):






																				"""simple docstring"""

																				snake_case_							 =						"AutoFeatureExtractor"
																				snake_case_							 =						"AutoTokenizer"
																				snake_case_							 =						False

													try:
																			AutoConfig.register("custom"							,	a    )
																			AutoFeatureExtractor.register(a							,	a    )
																			AutoTokenizer.register(a							,	slow_tokenizer_class=a    )
																			AutoProcessor.register(a							,	a    )
																			# If remote code is not set, the default is to use local classes.
																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"    )
																			self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )
																			self.assertFalse(processor.special_attribute_present    )
																			self.assertFalse(processor.feature_extractor.special_attribute_present    )
																			self.assertFalse(processor.tokenizer.special_attribute_present    )

																			# If remote code is disabled, we load the local ones.
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )
																			self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )
																			self.assertFalse(processor.special_attribute_present    )
																			self.assertFalse(processor.feature_extractor.special_attribute_present    )
																			self.assertFalse(processor.tokenizer.special_attribute_present    )

																			# If remote is enabled, we load from the Hub.
																			SCREAMING_SNAKE_CASE__     :				Any   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )
																			self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )
																			self.assertTrue(processor.special_attribute_present    )
																			self.assertTrue(processor.feature_extractor.special_attribute_present    )
																			self.assertTrue(processor.tokenizer.special_attribute_present    )

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in PROCESSOR_MAPPING._extra_content:
																									del PROCESSOR_MAPPING._extra_content[CustomConfig]

							def      A_  (      self				:     Optional[Any]    )	->Dict:
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert"    )
													self.assertEqual(processor.__class__.__name__							,	"BertTokenizerFast"    )






							def      A_  (      self				:     Dict    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				Dict   =					AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext"    )
													self.assertEqual(processor.__class__.__name__							,	"ConvNextImageProcessor"    )







@is_staging_test
class    _a     (				unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

							@classmethod
							def      A_  (      cls				:     List[str]    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				int   =					TOKEN
													HfFolder.save_token(a    )

							@classmethod
							def      A_  (      cls				:     List[str]    )	->Optional[int]:
													try:
																			delete_repo(token=cls._token							,	repo_id="test-processor"    )
													except HTTPError:
																			pass

													try:
																			delete_repo(token=cls._token							,	repo_id="valid_org/test-processor-org"    )
													except HTTPError:
																			pass

													try:
																			delete_repo(token=cls._token							,	repo_id="test-dynamic-processor"    )
													except HTTPError:
																			pass

							def      A_  (      self				:     Dict    )	->Dict:
													SCREAMING_SNAKE_CASE__     :				Tuple   =					WavaVecaProcessor.from_pretrained(a    )
													with tempfile.TemporaryDirectory() as tmp_dir:
																			processor.save_pretrained(
																			    os.path.join(a							,	"test-processor"    )							,	push_to_hub=a							,	use_auth_token=self._token    )

																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					WavaVecaProcessor.from_pretrained(f"""{USER}/test-processor"""    )
																			for k, v in processor.feature_extractor.__dict__.items():
																									self.assertEqual(a							,	getattr(new_processor.feature_extractor							,	a    )    )
																			self.assertDictEqual(new_processor.tokenizer.get_vocab()							,	processor.tokenizer.get_vocab()    )

							def      A_  (      self				:     List[str]    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					WavaVecaProcessor.from_pretrained(a    )

													with tempfile.TemporaryDirectory() as tmp_dir:
																			processor.save_pretrained(
																			    os.path.join(a							,	"test-processor-org"    )							,	push_to_hub=a							,	use_auth_token=self._token							,	organization="valid_org"							,	)

																			SCREAMING_SNAKE_CASE__     :				Dict   =					WavaVecaProcessor.from_pretrained("valid_org/test-processor-org"    )
																			for k, v in processor.feature_extractor.__dict__.items():
																									self.assertEqual(a							,	getattr(new_processor.feature_extractor							,	a    )    )
																			self.assertDictEqual(new_processor.tokenizer.get_vocab()							,	processor.tokenizer.get_vocab()    )






							def      A_  (      self				:     Any    )	->int:
													CustomFeatureExtractor.register_for_auto_class()
													CustomTokenizer.register_for_auto_class()
													CustomProcessor.register_for_auto_class()

													SCREAMING_SNAKE_CASE__     :				Any   =					CustomFeatureExtractor.from_pretrained(a    )

													with tempfile.TemporaryDirectory() as tmp_dir:
																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					os.path.join(a							,	"vocab.txt"    )
																			with open(a							,	"w"							,	encoding="utf-8"    ) as vocab_writer:
																									vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
																			SCREAMING_SNAKE_CASE__     :				str   =					CustomTokenizer(a    )

													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					CustomProcessor(a							,	a    )

													with tempfile.TemporaryDirectory() as tmp_dir:
																			create_repo(f"""{USER}/test-dynamic-processor"""							,	token=self._token    )
																			SCREAMING_SNAKE_CASE__     :				str   =					Repository(a							,	clone_from=f"""{USER}/test-dynamic-processor"""							,	token=self._token    )
																			processor.save_pretrained(a    )

																			# This has added the proper auto_map field to the feature extractor config
																			self.assertDictEqual(
																			    processor.feature_extractor.auto_map							,	{
																			        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
																			        "AutoProcessor": "custom_processing.CustomProcessor",
																			    }							,	)

																			# This has added the proper auto_map field to the tokenizer config
																			with open(os.path.join(a							,	"tokenizer_config.json"    )    ) as f:
																									SCREAMING_SNAKE_CASE__     :				str   =					json.load(a    )
																			self.assertDictEqual(
																			    tokenizer_config["auto_map"]							,	{
																			        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
																			        "AutoProcessor": "custom_processing.CustomProcessor",
																			    }							,	)

																			# The code has been copied from fixtures
																			self.assertTrue(os.path.isfile(os.path.join(a							,	"custom_feature_extraction.py"    )    )    )
																			self.assertTrue(os.path.isfile(os.path.join(a							,	"custom_tokenization.py"    )    )    )
																			self.assertTrue(os.path.isfile(os.path.join(a							,	"custom_processing.py"    )    )    )

																			repo.push_to_hub()

													SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained(f"""{USER}/test-dynamic-processor"""							,	trust_remote_code=a    )
													# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
													self.assertEqual(new_processor.__class__.__name__							,	"CustomProcessor"    )