
from typing import Dict

import numpy as np

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, GenericTensor, Pipeline, PipelineException


if is_tf_available():
					import tensorflow as tf

					from ..tf_utils import stable_softmax


if is_torch_available():
					import torch


_SCREAMING_SNAKE_CASE    :      Tuple          =			logging.get_logger(__name__)







@add_end_docstrings(
    SCREAMING_SNAKE_CASE_,			R"\n        top_k (`int`, defaults to 5):\n            The number of predictions to return.\n        targets (`str` or `List[str]`, *optional*):\n            When passed, the model will limit the scores to the passed targets instead of looking up in the whole\n            vocab. If the provided targets are not in the model vocab, they will be tokenized and the first resulting\n            token will be used (with a warning, and that might be slower).\n\n    ",			)
class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







	'''simple docstring'''





	def   __snake_case				(			self				,       UpperCamelCase__		):
					if self.framework == "tf":
									A__  :	List[str]         =							tf.where(input_ids == self.tokenizer.mask_token_id		).numpy()
					elif self.framework == "pt":
									A__  :	Any         =							torch.nonzero(input_ids == self.tokenizer.mask_token_id				,       as_tuple=UpperCamelCase__		)
					else:
									raise ValueError('''Unsupported framework'''		)
					return masked_index





	def   __snake_case				(			self				,       UpperCamelCase__		):
					A__  :	Dict         =							self.get_masked_index(UpperCamelCase__		)
					A__  :	str         =							np.prod(masked_index.shape		)
					if numel < 1:
									raise PipelineException(
									    '''fill-mask'''				,       self.model.base_model_prefix				,       F"No mask_token ({self.tokenizer.mask_token}) found on the input"				,       )





	def   __snake_case				(			self				,       UpperCamelCase__		):
					if isinstance(UpperCamelCase__				,       UpperCamelCase__		):
									for model_input in model_inputs:
													self._ensure_exactly_one_mask_token(model_input['''input_ids'''][0]		)
					else:
									for input_ids in model_inputs["input_ids"]:
													self._ensure_exactly_one_mask_token(UpperCamelCase__		)





	def   __snake_case				(			self				,       UpperCamelCase__				,       UpperCamelCase__=None				,       **UpperCamelCase__		):
					if return_tensors is None:
									A__  :	str         =							self.framework
					A__  :	str         =							self.tokenizer(UpperCamelCase__				,       return_tensors=UpperCamelCase__		)
					self.ensure_exactly_one_mask_token(UpperCamelCase__		)
					return model_inputs





	def   __snake_case				(			self				,       UpperCamelCase__		):
					A__  :	Tuple         =							self.model(**UpperCamelCase__		)
					A__  :	List[str]         =							model_inputs['''input_ids''']
					return model_outputs





	def   __snake_case				(			self				,       UpperCamelCase__				,       UpperCamelCase__=5				,       UpperCamelCase__=None		):
					# Cap top_k if there are targets
					if target_ids is not None and target_ids.shape[0] < top_k:
									A__  :	Union[str, Any]         =							target_ids.shape[0]
					A__  :	str         =							model_outputs['''input_ids'''][0]
					A__  :	Tuple         =							model_outputs['''logits''']

					if self.framework == "tf":
									A__  :	Tuple         =							tf.where(input_ids == self.tokenizer.mask_token_id		).numpy()[:, 0]

									A__  :	Dict         =							outputs.numpy()

									A__  :	Tuple         =							outputs[0, masked_index, :]
									A__  :	str         =							stable_softmax(UpperCamelCase__				,       axis=-1		)
									if target_ids is not None:
													A__  :	int         =							tf.gather_nd(tf.squeeze(UpperCamelCase__				,       0		)				,       target_ids.reshape(-1				,       1		)		)
													A__  :	str         =							tf.expand_dims(UpperCamelCase__				,       0		)

									A__  :	str         =							tf.math.top_k(UpperCamelCase__				,       k=UpperCamelCase__		)
									A__  :	List[str]         =							topk.values.numpy(), topk.indices.numpy()
					else:
									A__  :	Optional[Any]         =							torch.nonzero(input_ids == self.tokenizer.mask_token_id				,       as_tuple=UpperCamelCase__		).squeeze(-1		)
									# Fill mask pipeline supports only one ${mask_token} per sample

									A__  :	List[Any]         =							outputs[0, masked_index, :]
									A__  :	Tuple         =							logits.softmax(dim=-1		)
									if target_ids is not None:
													A__  :	str         =							probs[..., target_ids]

									A__  :	Tuple         =							probs.topk(UpperCamelCase__		)

					A__  :	Union[str, Any]         =							[]
					A__  :	str         =							values.shape[0] == 1
					for i, (_values, _predictions) in enumerate(zip(values.tolist()				,       predictions.tolist()		)		):
									A__  :	Optional[Any]         =							[]
									for v, p in zip(_values				,       _predictions		):
													# Copy is important since we're going to modify this array in place
													A__  :	List[str]         =							input_ids.numpy().copy()
													if target_ids is not None:
																	A__  :	Any         =							target_ids[p].tolist()

													A__  :	List[str]         =							p
													# Filter padding out:
													A__  :	Dict         =							tokens[np.where(tokens != self.tokenizer.pad_token_id		)]
													# Originally we skip special tokens to give readable output.
													# For multi masks though, the other [MASK] would be removed otherwise
													# making the output look odd, so we add them back
													A__  :	Any         =							self.tokenizer.decode(UpperCamelCase__				,       skip_special_tokens=UpperCamelCase__		)
													A__  :	str         =							{'''score''': v, '''token''': p, '''token_str''': self.tokenizer.decode([p]		), '''sequence''': sequence}
													row.append(UpperCamelCase__		)
									result.append(UpperCamelCase__		)
					if single_mask:
									return result[0]
					return result





	def   __snake_case				(			self				,       UpperCamelCase__				,       UpperCamelCase__=None		):
					if isinstance(UpperCamelCase__				,       UpperCamelCase__		):
									A__  :	Dict         =							[targets]
					try:
									A__  :	Any         =							self.tokenizer.get_vocab()
					except Exception:
									A__  :	Tuple         =							{}
					A__  :	str         =							[]
					for target in targets:
									A__  :	int         =							vocab.get(UpperCamelCase__				,       UpperCamelCase__		)
									if id_ is None:
													A__  :	Tuple         =							self.tokenizer(
													    UpperCamelCase__				,       add_special_tokens=UpperCamelCase__				,       return_attention_mask=UpperCamelCase__				,       return_token_type_ids=UpperCamelCase__				,       max_length=1				,       truncation=UpperCamelCase__				,       )['''input_ids''']
													if len(UpperCamelCase__		) == 0:
																	logger.warning(
																	    F"The specified target token `{target}` does not exist in the model vocabulary. "
																	    '''We cannot replace it with anything meaningful, ignoring it'''		)
																	continue
													A__  :	int         =							input_ids[0]
													# XXX: If users encounter this pass
													# it becomes pretty slow, so let's make sure
													# The warning enables them to fix the input to
													# get faster performance.
													logger.warning(
													    F"The specified target token `{target}` does not exist in the model vocabulary. "
													    F"Replacing with `{self.tokenizer.convert_ids_to_tokens(id_		)}`."		)
									target_ids.append(id_		)
					A__  :	str         =							list(set(UpperCamelCase__		)		)
					if len(UpperCamelCase__		) == 0:
									raise ValueError('''At least one target must be provided when passed.'''		)
					A__  :	Optional[int]         =							np.array(UpperCamelCase__		)
					return target_ids





	def   __snake_case				(			self				,       UpperCamelCase__=None				,       UpperCamelCase__=None		):
					A__  :	Tuple         =							{}

					if targets is not None:
									A__  :	Any         =							self.get_target_ids(UpperCamelCase__				,       UpperCamelCase__		)
									A__  :	List[Any]         =							target_ids

					if top_k is not None:
									A__  :	Union[str, Any]         =							top_k

					if self.tokenizer.mask_token_id is None:
									raise PipelineException(
									    '''fill-mask'''				,       self.model.base_model_prefix				,       '''The tokenizer does not define a `mask_token`.'''		)
					return {}, {}, postprocess_params





	def __call__(			self				,       UpperCamelCase__				,       *UpperCamelCase__				,       **UpperCamelCase__		):
					A__  :	Any         =							super().__call__(UpperCamelCase__				,       **UpperCamelCase__		)
					if isinstance(UpperCamelCase__				,       UpperCamelCase__		) and len(UpperCamelCase__		) == 1:
									return outputs[0]
					return outputs

import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_SCREAMING_SNAKE_CASE    :      List[Any]          =			get_tests_dir('fixtures/dummy_feature_extractor_config.json')
_SCREAMING_SNAKE_CASE    :      int          =			get_tests_dir('fixtures/vocab.json')
_SCREAMING_SNAKE_CASE    :      Tuple          =			get_tests_dir('fixtures')







class  UpperCamelCase__		(    unittest.TestCase   ):







	'''simple docstring'''
	_lowerCAmelCase							       =				["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





	def   __snake_case				(			self		):
					A__  :	List[Any]         =							0





	def   __snake_case				(			self		):
					A__  :	Dict         =							AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''		)
					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Optional[Any]         =							WavaVecaConfig()
									A__  :	Dict         =							AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''		)

									# save in new folder
									model_config.save_pretrained(UpperCamelCase__		)
									processor.save_pretrained(UpperCamelCase__		)

									A__  :	Any         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									# copy relevant files
									copyfile(UpperCamelCase__				,       os.path.join(UpperCamelCase__				,       UpperCamelCase__		)		)
									copyfile(UpperCamelCase__				,       os.path.join(UpperCamelCase__				,       '''vocab.json'''		)		)

									A__  :	List[Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Dict         =							WavaVecaFeatureExtractor()
									A__  :	Union[str, Any]         =							AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''		)

									A__  :	Optional[int]         =							WavaVecaProcessor(UpperCamelCase__				,       UpperCamelCase__		)

									# save in new folder
									processor.save_pretrained(UpperCamelCase__		)

									# drop `processor_class` in tokenizer
									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''r'''		) as f:
													A__  :	str         =							json.load(UpperCamelCase__		)
													config_dict.pop('''processor_class'''		)

									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''w'''		) as f:
													f.write(json.dumps(UpperCamelCase__		)		)

									A__  :	Optional[int]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Optional[int]         =							WavaVecaFeatureExtractor()
									A__  :	List[Any]         =							AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''		)

									A__  :	str         =							WavaVecaProcessor(UpperCamelCase__				,       UpperCamelCase__		)

									# save in new folder
									processor.save_pretrained(UpperCamelCase__		)

									# drop `processor_class` in feature extractor
									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''r'''		) as f:
													A__  :	List[Any]         =							json.load(UpperCamelCase__		)
													config_dict.pop('''processor_class'''		)

									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''w'''		) as f:
													f.write(json.dumps(UpperCamelCase__		)		)

									A__  :	List[Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Any         =							WavaVecaConfig(processor_class='''Wav2Vec2Processor'''		)
									model_config.save_pretrained(UpperCamelCase__		)
									# copy relevant files
									copyfile(UpperCamelCase__				,       os.path.join(UpperCamelCase__				,       '''vocab.json'''		)		)
									# create emtpy sample processor
									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''w'''		) as f:
													f.write('''{}'''		)

									A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					# If remote code is not set, we will time out when asking whether to load the model.
					with self.assertRaises(UpperCamelCase__		):
									A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''		)
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(UpperCamelCase__		):
									A__  :	str         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)

					A__  :	int         =							AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)
					self.assertTrue(processor.special_attribute_present		)
					self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)

					A__  :	List[Any]         =							processor.feature_extractor
					self.assertTrue(feature_extractor.special_attribute_present		)
					self.assertEqual(feature_extractor.__class__.__name__				,       '''NewFeatureExtractor'''		)

					A__  :	List[Any]         =							processor.tokenizer
					self.assertTrue(tokenizer.special_attribute_present		)
					if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__				,       '''NewTokenizerFast'''		)

									# Test we can also load the slow version
									A__  :	Dict         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__				,       use_fast=UpperCamelCase__		)
									A__  :	int         =							new_processor.tokenizer
									self.assertTrue(new_tokenizer.special_attribute_present		)
									self.assertEqual(new_tokenizer.__class__.__name__				,       '''NewTokenizer'''		)
					else:
									self.assertEqual(tokenizer.__class__.__name__				,       '''NewTokenizer'''		)





	def   __snake_case				(			self		):
					try:
									AutoConfig.register('''custom'''				,       UpperCamelCase__		)
									AutoFeatureExtractor.register(UpperCamelCase__				,       UpperCamelCase__		)
									AutoTokenizer.register(UpperCamelCase__				,       slow_tokenizer_class=UpperCamelCase__		)
									AutoProcessor.register(UpperCamelCase__				,       UpperCamelCase__		)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(UpperCamelCase__		):
													AutoProcessor.register(UpperCamelCase__				,       UpperCamelCase__		)

									# Now that the config is registered, it can be used as any other config with the auto-API
									A__  :	Any         =							CustomFeatureExtractor.from_pretrained(UpperCamelCase__		)

									with tempfile.TemporaryDirectory() as tmp_dir:
													A__  :	str         =							os.path.join(UpperCamelCase__				,       '''vocab.txt'''		)
													with open(UpperCamelCase__				,       '''w'''				,       encoding='''utf-8'''		) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
													A__  :	str         =							CustomTokenizer(UpperCamelCase__		)

									A__  :	Optional[Any]         =							CustomProcessor(UpperCamelCase__				,       UpperCamelCase__		)

									with tempfile.TemporaryDirectory() as tmp_dir:
													processor.save_pretrained(UpperCamelCase__		)
													A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)
													self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
									if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]





	def   __snake_case				(			self		):







					class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







						'''simple docstring'''
						_lowerCAmelCase							       =				False







					class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







						'''simple docstring'''
						_lowerCAmelCase							       =				False







					class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







						'''simple docstring'''
						_lowerCAmelCase							       =				"AutoFeatureExtractor"
						_lowerCAmelCase							       =				"AutoTokenizer"
						_lowerCAmelCase							       =				False

					try:
									AutoConfig.register('''custom'''				,       UpperCamelCase__		)
									AutoFeatureExtractor.register(UpperCamelCase__				,       UpperCamelCase__		)
									AutoTokenizer.register(UpperCamelCase__				,       slow_tokenizer_class=UpperCamelCase__		)
									AutoProcessor.register(UpperCamelCase__				,       UpperCamelCase__		)
									# If remote code is not set, the default is to use local classes.
									A__  :	List[Any]         =							AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''		)
									self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)
									self.assertFalse(processor.special_attribute_present		)
									self.assertFalse(processor.feature_extractor.special_attribute_present		)
									self.assertFalse(processor.tokenizer.special_attribute_present		)

									# If remote code is disabled, we load the local ones.
									A__  :	Any         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)
									self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)
									self.assertFalse(processor.special_attribute_present		)
									self.assertFalse(processor.feature_extractor.special_attribute_present		)
									self.assertFalse(processor.tokenizer.special_attribute_present		)

									# If remote is enabled, we load from the Hub.
									A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)
									self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)
									self.assertTrue(processor.special_attribute_present		)
									self.assertTrue(processor.feature_extractor.special_attribute_present		)
									self.assertTrue(processor.tokenizer.special_attribute_present		)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
									if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]





	def   __snake_case				(			self		):
					A__  :	str         =							AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
					self.assertEqual(processor.__class__.__name__				,       '''BertTokenizerFast'''		)





	def   __snake_case				(			self		):
					A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-convnext'''		)
					self.assertEqual(processor.__class__.__name__				,       '''ConvNextImageProcessor'''		)







@is_staging_test
class  UpperCamelCase__		(    unittest.TestCase   ):







	'''simple docstring'''
	_lowerCAmelCase							       =				["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





	@classmethod
	def   __snake_case				(			cls		):
					A__  :	List[str]         =							TOKEN
					HfFolder.save_token(UpperCamelCase__		)





	@classmethod
	def   __snake_case				(			cls		):
					try:
									delete_repo(token=cls._token				,       repo_id='''test-processor'''		)
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token				,       repo_id='''valid_org/test-processor-org'''		)
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token				,       repo_id='''test-dynamic-processor'''		)
					except HTTPError:
									pass





	def   __snake_case				(			self		):
					A__  :	Optional[Any]         =							WavaVecaProcessor.from_pretrained(UpperCamelCase__		)
					with tempfile.TemporaryDirectory() as tmp_dir:
									processor.save_pretrained(
									    os.path.join(UpperCamelCase__				,       '''test-processor'''		)				,       push_to_hub=UpperCamelCase__				,       use_auth_token=self._token		)

									A__  :	List[Any]         =							WavaVecaProcessor.from_pretrained(F"{USER}/test-processor"		)
									for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(UpperCamelCase__				,       getattr(new_processor.feature_extractor				,       UpperCamelCase__		)		)
									self.assertDictEqual(new_processor.tokenizer.get_vocab()				,       processor.tokenizer.get_vocab()		)





	def   __snake_case				(			self		):
					A__  :	int         =							WavaVecaProcessor.from_pretrained(UpperCamelCase__		)

					with tempfile.TemporaryDirectory() as tmp_dir:
									processor.save_pretrained(
									    os.path.join(UpperCamelCase__				,       '''test-processor-org'''		)				,       push_to_hub=UpperCamelCase__				,       use_auth_token=self._token				,       organization='''valid_org'''				,       )

									A__  :	List[str]         =							WavaVecaProcessor.from_pretrained('''valid_org/test-processor-org'''		)
									for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(UpperCamelCase__				,       getattr(new_processor.feature_extractor				,       UpperCamelCase__		)		)
									self.assertDictEqual(new_processor.tokenizer.get_vocab()				,       processor.tokenizer.get_vocab()		)





	def   __snake_case				(			self		):
					CustomFeatureExtractor.register_for_auto_class()
					CustomTokenizer.register_for_auto_class()
					CustomProcessor.register_for_auto_class()

					A__  :	Optional[Any]         =							CustomFeatureExtractor.from_pretrained(UpperCamelCase__		)

					with tempfile.TemporaryDirectory() as tmp_dir:
									A__  :	List[Any]         =							os.path.join(UpperCamelCase__				,       '''vocab.txt'''		)
									with open(UpperCamelCase__				,       '''w'''				,       encoding='''utf-8'''		) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
									A__  :	Union[str, Any]         =							CustomTokenizer(UpperCamelCase__		)

					A__  :	List[Any]         =							CustomProcessor(UpperCamelCase__				,       UpperCamelCase__		)

					with tempfile.TemporaryDirectory() as tmp_dir:
									create_repo(F"{USER}/test-dynamic-processor"				,       token=self._token		)
									A__  :	Union[str, Any]         =							Repository(UpperCamelCase__				,       clone_from=F"{USER}/test-dynamic-processor"				,       token=self._token		)
									processor.save_pretrained(UpperCamelCase__		)

									# This has added the proper auto_map field to the feature extractor config
									self.assertDictEqual(
									    processor.feature_extractor.auto_map				,       {
									        '''AutoFeatureExtractor''': '''custom_feature_extraction.CustomFeatureExtractor''',
									        '''AutoProcessor''': '''custom_processing.CustomProcessor''',
									    }				,       )

									# This has added the proper auto_map field to the tokenizer config
									with open(os.path.join(UpperCamelCase__				,       '''tokenizer_config.json'''		)		) as f:
													A__  :	Optional[int]         =							json.load(UpperCamelCase__		)
									self.assertDictEqual(
									    tokenizer_config['''auto_map''']				,       {
									        '''AutoTokenizer''': ['''custom_tokenization.CustomTokenizer''', None],
									        '''AutoProcessor''': '''custom_processing.CustomProcessor''',
									    }				,       )

									# The code has been copied from fixtures
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__				,       '''custom_feature_extraction.py'''		)		)		)
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__				,       '''custom_tokenization.py'''		)		)		)
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__				,       '''custom_processing.py'''		)		)		)

									repo.push_to_hub()

					A__  :	Tuple         =							AutoProcessor.from_pretrained(F"{USER}/test-dynamic-processor"				,       trust_remote_code=UpperCamelCase__		)
					# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
					self.assertEqual(new_processor.__class__.__name__				,       '''CustomProcessor'''		)