
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE    :      str          =			get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
					from transformers.models.mbart.modeling_mbart import shift_tokens_right

_SCREAMING_SNAKE_CASE    :      Optional[Any]          =			2_5_0_0_0_4
_SCREAMING_SNAKE_CASE    :      Optional[Any]          =			2_5_0_0_2_0







@require_sentencepiece
@require_tokenizers
class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_,			unittest.TestCase   ):







	'''simple docstring'''
	_lowerCAmelCase							       =				MBartTokenizer
	_lowerCAmelCase							       =				MBartTokenizerFast
	_lowerCAmelCase							       =				True
	_lowerCAmelCase							       =				True





	def   __snake_case				(			self		):
					super().setUp()

					# We have a SentencePiece fixture for testing
					A__  :	str         =							MBartTokenizer(UpperCamelCase__				,       keep_accents=UpperCamelCase__		)
					tokenizer.save_pretrained(self.tmpdirname		)





	def   __snake_case				(			self		):
					A__  :	List[str]         =							MBartTokenizer(UpperCamelCase__				,       keep_accents=UpperCamelCase__		)

					A__  :	Optional[Any]         =							tokenizer.tokenize('''This is a test'''		)
					self.assertListEqual(UpperCamelCase__				,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(UpperCamelCase__		)				,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				,       )

					A__  :	List[Any]         =							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
					self.assertListEqual(
					    UpperCamelCase__				,       [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''é''',
					        '''.''',
					    ]				,       )
					A__  :	Optional[int]         =							tokenizer.convert_tokens_to_ids(UpperCamelCase__		)
					self.assertListEqual(
					    UpperCamelCase__				,       [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
					    ]				,       )

					A__  :	Optional[Any]         =							tokenizer.convert_ids_to_tokens(UpperCamelCase__		)
					self.assertListEqual(
					    UpperCamelCase__				,       [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''<unk>''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''<unk>''',
					        '''.''',
					    ]				,       )





	def   __snake_case				(			self		):
					if not self.test_slow_tokenizer:
									# as we don't have a slow version, we can't compare the outputs between slow and fast versions
									return

					A__  :	Optional[int]         =							(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart''', {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"		):
													A__  :	Tuple         =							self.rust_tokenizer_class.from_pretrained(UpperCamelCase__				,       **UpperCamelCase__		)
													A__  :	Tuple         =							self.tokenizer_class.from_pretrained(UpperCamelCase__				,       **UpperCamelCase__		)

													A__  :	int         =							tempfile.mkdtemp()

													A__  :	List[str]         =							tokenizer_r.save_pretrained(UpperCamelCase__		)
													A__  :	str         =							tokenizer_p.save_pretrained(UpperCamelCase__		)

													# Checks it save with the same files + the tokenizer.json file for the fast one
													self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)
													A__  :	List[Any]         =							tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f		)
													self.assertSequenceEqual(UpperCamelCase__				,       UpperCamelCase__		)

													# Checks everything loads correctly in the same way
													A__  :	Optional[Any]         =							tokenizer_r.from_pretrained(UpperCamelCase__		)
													A__  :	Dict         =							tokenizer_p.from_pretrained(UpperCamelCase__		)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(UpperCamelCase__				,       UpperCamelCase__		)		)
																	# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																	# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

													shutil.rmtree(UpperCamelCase__		)

													# Save tokenizer rust, legacy_format=True
													A__  :	List[str]         =							tempfile.mkdtemp()

													A__  :	List[Any]         =							tokenizer_r.save_pretrained(UpperCamelCase__				,       legacy_format=UpperCamelCase__		)
													A__  :	Any         =							tokenizer_p.save_pretrained(UpperCamelCase__		)

													# Checks it save with the same files
													self.assertSequenceEqual(UpperCamelCase__				,       UpperCamelCase__		)

													# Checks everything loads correctly in the same way
													A__  :	str         =							tokenizer_r.from_pretrained(UpperCamelCase__		)
													A__  :	Tuple         =							tokenizer_p.from_pretrained(UpperCamelCase__		)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(UpperCamelCase__				,       UpperCamelCase__		)		)

													shutil.rmtree(UpperCamelCase__		)

													# Save tokenizer rust, legacy_format=False
													A__  :	Optional[Any]         =							tempfile.mkdtemp()

													A__  :	Dict         =							tokenizer_r.save_pretrained(UpperCamelCase__				,       legacy_format=UpperCamelCase__		)
													A__  :	List[str]         =							tokenizer_p.save_pretrained(UpperCamelCase__		)

													# Checks it saved the tokenizer.json file
													self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)

													# Checks everything loads correctly in the same way
													A__  :	List[Any]         =							tokenizer_r.from_pretrained(UpperCamelCase__		)
													A__  :	List[Any]         =							tokenizer_p.from_pretrained(UpperCamelCase__		)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(UpperCamelCase__				,       UpperCamelCase__		)		)

													shutil.rmtree(UpperCamelCase__		)







@require_torch
@require_sentencepiece
@require_tokenizers
class  UpperCamelCase__		(    unittest.TestCase   ):







	'''simple docstring'''
	_lowerCAmelCase							       =				"facebook/mbart-large-en-ro"
	_lowerCAmelCase							       =				[
	    " UN Chief Says There Is No Military Solution in Syria",
	    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
	]
	_lowerCAmelCase							       =				[
	    "Şeful ONU declară că nu există o soluţie militară în Siria",
	    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
	    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
	    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
	]
	_lowerCAmelCase							       =				[8_274, 127_873, 25_916, 7, 8_622, 2_071, 438, 67_485, 53, 187_895, 23, 51_712, 2, EN_CODE]





	@classmethod
	def   __snake_case				(			cls		):
					A__  :	MBartTokenizer         =							MBartTokenizer.from_pretrained(
					    cls.checkpoint_name				,       src_lang='''en_XX'''				,       tgt_lang='''ro_RO'''		)
					A__  :	List[Any]         =							1
					return cls





	def   __snake_case				(			self		):
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']				,       25_0001		)
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']				,       25_0004		)
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']				,       25_0020		)





	def   __snake_case				(			self		):
					A__  :	Any         =							self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
					self.assertListEqual(self.expected_src_tokens				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					self.assertIn(UpperCamelCase__				,       self.tokenizer.all_special_ids		)
					A__  :	List[str]         =							[RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
					A__  :	Union[str, Any]         =							self.tokenizer.decode(UpperCamelCase__				,       skip_special_tokens=UpperCamelCase__		)
					A__  :	Tuple         =							self.tokenizer.decode(generated_ids[1:]				,       skip_special_tokens=UpperCamelCase__		)
					self.assertEqual(UpperCamelCase__				,       UpperCamelCase__		)
					self.assertNotIn(self.tokenizer.eos_token				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					A__  :	Tuple         =							['''this is gunna be a long sentence ''' * 20]
					assert isinstance(src_text[0]				,       UpperCamelCase__		)
					A__  :	List[str]         =							10
					A__  :	Optional[Any]         =							self.tokenizer(UpperCamelCase__				,       max_length=UpperCamelCase__				,       truncation=UpperCamelCase__		).input_ids[0]
					self.assertEqual(ids[-2]				,       2		)
					self.assertEqual(ids[-1]				,       UpperCamelCase__		)
					self.assertEqual(len(UpperCamelCase__		)				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']		)				,       [25_0026, 25_0001]		)





	def   __snake_case				(			self		):
					A__  :	Optional[int]         =							tempfile.mkdtemp()
					A__  :	Any         =							self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(UpperCamelCase__		)
					A__  :	Any         =							MBartTokenizer.from_pretrained(UpperCamelCase__		)
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,       UpperCamelCase__		)





	@require_torch
	def   __snake_case				(			self		):
					A__  :	Tuple         =							self.tokenizer(self.src_text				,       text_target=self.tgt_text				,       padding=UpperCamelCase__				,       return_tensors='''pt'''		)
					A__  :	Any         =							shift_tokens_right(batch['''labels''']				,       self.tokenizer.pad_token_id		)

					# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
					assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
					assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
					assert batch.decoder_input_ids[1][-1] == 2
					assert batch.labels[1][-2:].tolist() == [2, RO_CODE]





	@require_torch
	def   __snake_case				(			self		):
					A__  :	int         =							self.tokenizer(
					    self.src_text				,       text_target=self.tgt_text				,       padding=UpperCamelCase__				,       truncation=UpperCamelCase__				,       max_length=len(self.expected_src_tokens		)				,       return_tensors='''pt'''				,       )

					A__  :	List[str]         =							shift_tokens_right(batch['''labels''']				,       self.tokenizer.pad_token_id		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)

					self.assertEqual((2, 14)				,       batch.input_ids.shape		)
					self.assertEqual((2, 14)				,       batch.attention_mask.shape		)
					A__  :	Optional[Any]         =							batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens				,       UpperCamelCase__		)
					self.assertEqual(2				,       batch.decoder_input_ids[0, -1]		)  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens				,       []		)
					self.assertEqual(self.tokenizer.suffix_tokens				,       [self.tokenizer.eos_token_id, EN_CODE]		)





	def   __snake_case				(			self		):
					A__  :	Union[str, Any]         =							self.tokenizer(self.src_text				,       padding=UpperCamelCase__				,       truncation=UpperCamelCase__				,       max_length=3				,       return_tensors='''pt'''		)
					A__  :	Tuple         =							self.tokenizer(
					    text_target=self.tgt_text				,       padding=UpperCamelCase__				,       truncation=UpperCamelCase__				,       max_length=10				,       return_tensors='''pt'''		)
					A__  :	Union[str, Any]         =							targets['''input_ids''']
					A__  :	Any         =							shift_tokens_right(UpperCamelCase__				,       self.tokenizer.pad_token_id		)

					self.assertEqual(batch.input_ids.shape[1]				,       3		)
					self.assertEqual(batch.decoder_input_ids.shape[1]				,       10		)





	@require_torch
	def   __snake_case				(			self		):
					A__  :	Optional[int]         =							self.tokenizer._build_translation_inputs(
					    '''A test'''				,       return_tensors='''pt'''				,       src_lang='''en_XX'''				,       tgt_lang='''ar_AR'''		)

					self.assertEqual(
					    nested_simplify(UpperCamelCase__		)				,       {
					        # A, test, EOS, en_XX
					        '''input_ids''': [[62, 3034, 2, 25_0004]],
					        '''attention_mask''': [[1, 1, 1, 1]],
					        # ar_AR
					        '''forced_bos_token_id''': 25_0001,
					    }				,       )

import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_SCREAMING_SNAKE_CASE    :      List[Any]          =			get_tests_dir('fixtures/dummy_feature_extractor_config.json')
_SCREAMING_SNAKE_CASE    :      int          =			get_tests_dir('fixtures/vocab.json')
_SCREAMING_SNAKE_CASE    :      Tuple          =			get_tests_dir('fixtures')







class  UpperCamelCase__		(    unittest.TestCase   ):







	'''simple docstring'''
	_lowerCAmelCase							       =				["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





	def   __snake_case				(			self		):
					A__  :	List[Any]         =							0





	def   __snake_case				(			self		):
					A__  :	Dict         =							AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''		)
					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Optional[Any]         =							WavaVecaConfig()
									A__  :	Dict         =							AutoProcessor.from_pretrained('''facebook/wav2vec2-base-960h'''		)

									# save in new folder
									model_config.save_pretrained(UpperCamelCase__		)
									processor.save_pretrained(UpperCamelCase__		)

									A__  :	Any         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									# copy relevant files
									copyfile(UpperCamelCase__				,       os.path.join(UpperCamelCase__				,       UpperCamelCase__		)		)
									copyfile(UpperCamelCase__				,       os.path.join(UpperCamelCase__				,       '''vocab.json'''		)		)

									A__  :	List[Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Dict         =							WavaVecaFeatureExtractor()
									A__  :	Union[str, Any]         =							AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''		)

									A__  :	Optional[int]         =							WavaVecaProcessor(UpperCamelCase__				,       UpperCamelCase__		)

									# save in new folder
									processor.save_pretrained(UpperCamelCase__		)

									# drop `processor_class` in tokenizer
									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''r'''		) as f:
													A__  :	str         =							json.load(UpperCamelCase__		)
													config_dict.pop('''processor_class'''		)

									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''w'''		) as f:
													f.write(json.dumps(UpperCamelCase__		)		)

									A__  :	Optional[int]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Optional[int]         =							WavaVecaFeatureExtractor()
									A__  :	List[Any]         =							AutoTokenizer.from_pretrained('''facebook/wav2vec2-base-960h'''		)

									A__  :	str         =							WavaVecaProcessor(UpperCamelCase__				,       UpperCamelCase__		)

									# save in new folder
									processor.save_pretrained(UpperCamelCase__		)

									# drop `processor_class` in feature extractor
									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''r'''		) as f:
													A__  :	List[Any]         =							json.load(UpperCamelCase__		)
													config_dict.pop('''processor_class'''		)

									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''w'''		) as f:
													f.write(json.dumps(UpperCamelCase__		)		)

									A__  :	List[Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					with tempfile.TemporaryDirectory() as tmpdirname:
									A__  :	Any         =							WavaVecaConfig(processor_class='''Wav2Vec2Processor'''		)
									model_config.save_pretrained(UpperCamelCase__		)
									# copy relevant files
									copyfile(UpperCamelCase__				,       os.path.join(UpperCamelCase__				,       '''vocab.json'''		)		)
									# create emtpy sample processor
									with open(os.path.join(UpperCamelCase__				,       UpperCamelCase__		)				,       '''w'''		) as f:
													f.write('''{}'''		)

									A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)

					self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)





	def   __snake_case				(			self		):
					# If remote code is not set, we will time out when asking whether to load the model.
					with self.assertRaises(UpperCamelCase__		):
									A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''		)
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(UpperCamelCase__		):
									A__  :	str         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)

					A__  :	int         =							AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)
					self.assertTrue(processor.special_attribute_present		)
					self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)

					A__  :	List[Any]         =							processor.feature_extractor
					self.assertTrue(feature_extractor.special_attribute_present		)
					self.assertEqual(feature_extractor.__class__.__name__				,       '''NewFeatureExtractor'''		)

					A__  :	List[Any]         =							processor.tokenizer
					self.assertTrue(tokenizer.special_attribute_present		)
					if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__				,       '''NewTokenizerFast'''		)

									# Test we can also load the slow version
									A__  :	Dict         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__				,       use_fast=UpperCamelCase__		)
									A__  :	int         =							new_processor.tokenizer
									self.assertTrue(new_tokenizer.special_attribute_present		)
									self.assertEqual(new_tokenizer.__class__.__name__				,       '''NewTokenizer'''		)
					else:
									self.assertEqual(tokenizer.__class__.__name__				,       '''NewTokenizer'''		)





	def   __snake_case				(			self		):
					try:
									AutoConfig.register('''custom'''				,       UpperCamelCase__		)
									AutoFeatureExtractor.register(UpperCamelCase__				,       UpperCamelCase__		)
									AutoTokenizer.register(UpperCamelCase__				,       slow_tokenizer_class=UpperCamelCase__		)
									AutoProcessor.register(UpperCamelCase__				,       UpperCamelCase__		)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(UpperCamelCase__		):
													AutoProcessor.register(UpperCamelCase__				,       UpperCamelCase__		)

									# Now that the config is registered, it can be used as any other config with the auto-API
									A__  :	Any         =							CustomFeatureExtractor.from_pretrained(UpperCamelCase__		)

									with tempfile.TemporaryDirectory() as tmp_dir:
													A__  :	str         =							os.path.join(UpperCamelCase__				,       '''vocab.txt'''		)
													with open(UpperCamelCase__				,       '''w'''				,       encoding='''utf-8'''		) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
													A__  :	str         =							CustomTokenizer(UpperCamelCase__		)

									A__  :	Optional[Any]         =							CustomProcessor(UpperCamelCase__				,       UpperCamelCase__		)

									with tempfile.TemporaryDirectory() as tmp_dir:
													processor.save_pretrained(UpperCamelCase__		)
													A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained(UpperCamelCase__		)
													self.assertIsInstance(UpperCamelCase__				,       UpperCamelCase__		)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
									if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]





	def   __snake_case				(			self		):







					class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







						'''simple docstring'''
						_lowerCAmelCase							       =				False







					class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







						'''simple docstring'''
						_lowerCAmelCase							       =				False







					class  UpperCamelCase__		(    SCREAMING_SNAKE_CASE_   ):







						'''simple docstring'''
						_lowerCAmelCase							       =				"AutoFeatureExtractor"
						_lowerCAmelCase							       =				"AutoTokenizer"
						_lowerCAmelCase							       =				False

					try:
									AutoConfig.register('''custom'''				,       UpperCamelCase__		)
									AutoFeatureExtractor.register(UpperCamelCase__				,       UpperCamelCase__		)
									AutoTokenizer.register(UpperCamelCase__				,       slow_tokenizer_class=UpperCamelCase__		)
									AutoProcessor.register(UpperCamelCase__				,       UpperCamelCase__		)
									# If remote code is not set, the default is to use local classes.
									A__  :	List[Any]         =							AutoProcessor.from_pretrained('''hf-internal-testing/test_dynamic_processor'''		)
									self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)
									self.assertFalse(processor.special_attribute_present		)
									self.assertFalse(processor.feature_extractor.special_attribute_present		)
									self.assertFalse(processor.tokenizer.special_attribute_present		)

									# If remote code is disabled, we load the local ones.
									A__  :	Any         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)
									self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)
									self.assertFalse(processor.special_attribute_present		)
									self.assertFalse(processor.feature_extractor.special_attribute_present		)
									self.assertFalse(processor.tokenizer.special_attribute_present		)

									# If remote is enabled, we load from the Hub.
									A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained(
									    '''hf-internal-testing/test_dynamic_processor'''				,       trust_remote_code=UpperCamelCase__		)
									self.assertEqual(processor.__class__.__name__				,       '''NewProcessor'''		)
									self.assertTrue(processor.special_attribute_present		)
									self.assertTrue(processor.feature_extractor.special_attribute_present		)
									self.assertTrue(processor.tokenizer.special_attribute_present		)

					finally:
									if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
									if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]





	def   __snake_case				(			self		):
					A__  :	str         =							AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
					self.assertEqual(processor.__class__.__name__				,       '''BertTokenizerFast'''		)





	def   __snake_case				(			self		):
					A__  :	Union[str, Any]         =							AutoProcessor.from_pretrained('''hf-internal-testing/tiny-random-convnext'''		)
					self.assertEqual(processor.__class__.__name__				,       '''ConvNextImageProcessor'''		)







@is_staging_test
class  UpperCamelCase__		(    unittest.TestCase   ):







	'''simple docstring'''
	_lowerCAmelCase							       =				["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





	@classmethod
	def   __snake_case				(			cls		):
					A__  :	List[str]         =							TOKEN
					HfFolder.save_token(UpperCamelCase__		)





	@classmethod
	def   __snake_case				(			cls		):
					try:
									delete_repo(token=cls._token				,       repo_id='''test-processor'''		)
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token				,       repo_id='''valid_org/test-processor-org'''		)
					except HTTPError:
									pass

					try:
									delete_repo(token=cls._token				,       repo_id='''test-dynamic-processor'''		)
					except HTTPError:
									pass





	def   __snake_case				(			self		):
					A__  :	Optional[Any]         =							WavaVecaProcessor.from_pretrained(UpperCamelCase__		)
					with tempfile.TemporaryDirectory() as tmp_dir:
									processor.save_pretrained(
									    os.path.join(UpperCamelCase__				,       '''test-processor'''		)				,       push_to_hub=UpperCamelCase__				,       use_auth_token=self._token		)

									A__  :	List[Any]         =							WavaVecaProcessor.from_pretrained(F"{USER}/test-processor"		)
									for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(UpperCamelCase__				,       getattr(new_processor.feature_extractor				,       UpperCamelCase__		)		)
									self.assertDictEqual(new_processor.tokenizer.get_vocab()				,       processor.tokenizer.get_vocab()		)





	def   __snake_case				(			self		):
					A__  :	int         =							WavaVecaProcessor.from_pretrained(UpperCamelCase__		)

					with tempfile.TemporaryDirectory() as tmp_dir:
									processor.save_pretrained(
									    os.path.join(UpperCamelCase__				,       '''test-processor-org'''		)				,       push_to_hub=UpperCamelCase__				,       use_auth_token=self._token				,       organization='''valid_org'''				,       )

									A__  :	List[str]         =							WavaVecaProcessor.from_pretrained('''valid_org/test-processor-org'''		)
									for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(UpperCamelCase__				,       getattr(new_processor.feature_extractor				,       UpperCamelCase__		)		)
									self.assertDictEqual(new_processor.tokenizer.get_vocab()				,       processor.tokenizer.get_vocab()		)





	def   __snake_case				(			self		):
					CustomFeatureExtractor.register_for_auto_class()
					CustomTokenizer.register_for_auto_class()
					CustomProcessor.register_for_auto_class()

					A__  :	Optional[Any]         =							CustomFeatureExtractor.from_pretrained(UpperCamelCase__		)

					with tempfile.TemporaryDirectory() as tmp_dir:
									A__  :	List[Any]         =							os.path.join(UpperCamelCase__				,       '''vocab.txt'''		)
									with open(UpperCamelCase__				,       '''w'''				,       encoding='''utf-8'''		) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
									A__  :	Union[str, Any]         =							CustomTokenizer(UpperCamelCase__		)

					A__  :	List[Any]         =							CustomProcessor(UpperCamelCase__				,       UpperCamelCase__		)

					with tempfile.TemporaryDirectory() as tmp_dir:
									create_repo(F"{USER}/test-dynamic-processor"				,       token=self._token		)
									A__  :	Union[str, Any]         =							Repository(UpperCamelCase__				,       clone_from=F"{USER}/test-dynamic-processor"				,       token=self._token		)
									processor.save_pretrained(UpperCamelCase__		)

									# This has added the proper auto_map field to the feature extractor config
									self.assertDictEqual(
									    processor.feature_extractor.auto_map				,       {
									        '''AutoFeatureExtractor''': '''custom_feature_extraction.CustomFeatureExtractor''',
									        '''AutoProcessor''': '''custom_processing.CustomProcessor''',
									    }				,       )

									# This has added the proper auto_map field to the tokenizer config
									with open(os.path.join(UpperCamelCase__				,       '''tokenizer_config.json'''		)		) as f:
													A__  :	Optional[int]         =							json.load(UpperCamelCase__		)
									self.assertDictEqual(
									    tokenizer_config['''auto_map''']				,       {
									        '''AutoTokenizer''': ['''custom_tokenization.CustomTokenizer''', None],
									        '''AutoProcessor''': '''custom_processing.CustomProcessor''',
									    }				,       )

									# The code has been copied from fixtures
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__				,       '''custom_feature_extraction.py'''		)		)		)
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__				,       '''custom_tokenization.py'''		)		)		)
									self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__				,       '''custom_processing.py'''		)		)		)

									repo.push_to_hub()

					A__  :	Tuple         =							AutoProcessor.from_pretrained(F"{USER}/test-dynamic-processor"				,       trust_remote_code=UpperCamelCase__		)
					# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
					self.assertEqual(new_processor.__class__.__name__				,       '''CustomProcessor'''		)