



import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


lowercase_								=     get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
lowercase_								=     get_tests_dir("""fixtures/vocab.json""")
lowercase_								=     get_tests_dir("""fixtures""")
class 				_snake_case			(					unittest.TestCase):
				UpperCamelCase__				:					Tuple     		=["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				0




				def 							A__      (						self					:						List[Any]						):
							lowercase__					  =				AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)
							self.assertIsInstance(__lowercase,					__lowercase						)




				def 							A__      (						self					:						List[Any]						):
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowercase__					  =				WavaVecaConfig()
										lowercase__					  =				AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)

										# save in new folder
										model_config.save_pretrained(__lowercase						)
										processor.save_pretrained(__lowercase						)

										lowercase__					  =				AutoProcessor.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase,					__lowercase						)




				def 							A__      (						self					:						Optional[Any]						):
							with tempfile.TemporaryDirectory() as tmpdirname:
										# copy relevant files
										copyfile(__lowercase,					os.path.join(__lowercase,					__lowercase						)						)
										copyfile(__lowercase,					os.path.join(__lowercase,					"vocab.json"						)						)

										lowercase__					  =				AutoProcessor.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase,					__lowercase						)




				def 							A__      (						self					:						List[str]						):
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowercase__					  =				WavaVecaFeatureExtractor()
										lowercase__					  =				AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

										lowercase__					  =				WavaVecaProcessor(__lowercase,					__lowercase						)

										# save in new folder
										processor.save_pretrained(__lowercase						)

										# drop `processor_class` in tokenizer
										with open(os.path.join(__lowercase,					__lowercase						),					"r"						) as f:
													lowercase__					  =				json.load(__lowercase						)
													config_dict.pop("processor_class"						)

										with open(os.path.join(__lowercase,					__lowercase						),					"w"						) as f:
													f.write(json.dumps(__lowercase						)						)

										lowercase__					  =				AutoProcessor.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase,					__lowercase						)




				def 							A__      (						self					:						Dict						):
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowercase__					  =				WavaVecaFeatureExtractor()
										lowercase__					  =				AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

										lowercase__					  =				WavaVecaProcessor(__lowercase,					__lowercase						)

										# save in new folder
										processor.save_pretrained(__lowercase						)

										# drop `processor_class` in feature extractor
										with open(os.path.join(__lowercase,					__lowercase						),					"r"						) as f:
													lowercase__					  =				json.load(__lowercase						)
													config_dict.pop("processor_class"						)

										with open(os.path.join(__lowercase,					__lowercase						),					"w"						) as f:
													f.write(json.dumps(__lowercase						)						)

										lowercase__					  =				AutoProcessor.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase,					__lowercase						)




				def 							A__      (						self					:						Dict						):
							with tempfile.TemporaryDirectory() as tmpdirname:
										lowercase__					  =				WavaVecaConfig(processor_class="Wav2Vec2Processor"						)
										model_config.save_pretrained(__lowercase						)
										# copy relevant files
										copyfile(__lowercase,					os.path.join(__lowercase,					"vocab.json"						)						)
										# create emtpy sample processor
										with open(os.path.join(__lowercase,					__lowercase						),					"w"						) as f:
													f.write("{}"						)

										lowercase__					  =				AutoProcessor.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase,					__lowercase						)




				def 							A__      (						self					:						List[Any]						):
							# If remote code is not set, we will time out when asking whether to load the model.
							with self.assertRaises(__lowercase						):
										lowercase__					  =				AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__lowercase						):
										lowercase__					  =				AutoProcessor.from_pretrained(
										    "hf-internal-testing/test_dynamic_processor",					trust_remote_code=__lowercase						)

							lowercase__					  =				AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor",					trust_remote_code=__lowercase						)
							self.assertTrue(processor.special_attribute_present						)
							self.assertEqual(processor.__class__.__name__,					"NewProcessor"						)

							lowercase__					  =				processor.feature_extractor
							self.assertTrue(feature_extractor.special_attribute_present						)
							self.assertEqual(feature_extractor.__class__.__name__,					"NewFeatureExtractor"						)

							lowercase__					  =				processor.tokenizer
							self.assertTrue(tokenizer.special_attribute_present						)
							if is_tokenizers_available():
										self.assertEqual(tokenizer.__class__.__name__,					"NewTokenizerFast"						)

										# Test we can also load the slow version
										lowercase__					  =				AutoProcessor.from_pretrained(
										    "hf-internal-testing/test_dynamic_processor",					trust_remote_code=__lowercase,					use_fast=__lowercase						)
										lowercase__					  =				new_processor.tokenizer
										self.assertTrue(new_tokenizer.special_attribute_present						)
										self.assertEqual(new_tokenizer.__class__.__name__,					"NewTokenizer"						)
							else:
										self.assertEqual(tokenizer.__class__.__name__,					"NewTokenizer"						)




				def 							A__      (						self					:						int						):
							try:
										AutoConfig.register("custom",					__lowercase						)
										AutoFeatureExtractor.register(__lowercase,					__lowercase						)
										AutoTokenizer.register(__lowercase,					slow_tokenizer_class=__lowercase						)
										AutoProcessor.register(__lowercase,					__lowercase						)
										# Trying to register something existing in the Transformers library will raise an error
										with self.assertRaises(__lowercase						):
													AutoProcessor.register(__lowercase,					__lowercase						)

										# Now that the config is registered, it can be used as any other config with the auto-API
										lowercase__					  =				CustomFeatureExtractor.from_pretrained(__lowercase						)

										with tempfile.TemporaryDirectory() as tmp_dir:
													lowercase__					  =				os.path.join(__lowercase,					"vocab.txt"						)
													with open(__lowercase,					"w",					encoding="utf-8"						) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
													lowercase__					  =				CustomTokenizer(__lowercase						)

										lowercase__					  =				CustomProcessor(__lowercase,					__lowercase						)

										with tempfile.TemporaryDirectory() as tmp_dir:
													processor.save_pretrained(__lowercase						)
													lowercase__					  =				AutoProcessor.from_pretrained(__lowercase						)
													self.assertIsInstance(__lowercase,					__lowercase						)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
										if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]




				def 							A__      (						self					:						Dict						):
							class 				_snake_case			(					lowercase__):
											UpperCamelCase__				:					Union[str, Any]     		=False
							class 				_snake_case			(					lowercase__):
											UpperCamelCase__				:					Dict     		=False
							class 				_snake_case			(					lowercase__):
											UpperCamelCase__				:					Union[str, Any]     		="""AutoFeatureExtractor"""
											UpperCamelCase__				:					List[str]     		="""AutoTokenizer"""
											UpperCamelCase__				:					Union[str, Any]     		=False

							try:
										AutoConfig.register("custom",					__lowercase						)
										AutoFeatureExtractor.register(__lowercase,					__lowercase						)
										AutoTokenizer.register(__lowercase,					slow_tokenizer_class=__lowercase						)
										AutoProcessor.register(__lowercase,					__lowercase						)
										# If remote code is not set, the default is to use local classes.
										lowercase__					  =				AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
										self.assertEqual(processor.__class__.__name__,					"NewProcessor"						)
										self.assertFalse(processor.special_attribute_present						)
										self.assertFalse(processor.feature_extractor.special_attribute_present						)
										self.assertFalse(processor.tokenizer.special_attribute_present						)

										# If remote code is disabled, we load the local ones.
										lowercase__					  =				AutoProcessor.from_pretrained(
										    "hf-internal-testing/test_dynamic_processor",					trust_remote_code=__lowercase						)
										self.assertEqual(processor.__class__.__name__,					"NewProcessor"						)
										self.assertFalse(processor.special_attribute_present						)
										self.assertFalse(processor.feature_extractor.special_attribute_present						)
										self.assertFalse(processor.tokenizer.special_attribute_present						)

										# If remote is enabled, we load from the Hub.
										lowercase__					  =				AutoProcessor.from_pretrained(
										    "hf-internal-testing/test_dynamic_processor",					trust_remote_code=__lowercase						)
										self.assertEqual(processor.__class__.__name__,					"NewProcessor"						)
										self.assertTrue(processor.special_attribute_present						)
										self.assertTrue(processor.feature_extractor.special_attribute_present						)
										self.assertTrue(processor.tokenizer.special_attribute_present						)

							finally:
										if "custom" in CONFIG_MAPPING._extra_content:
													del CONFIG_MAPPING._extra_content["custom"]
										if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
													del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
										if CustomConfig in TOKENIZER_MAPPING._extra_content:
													del TOKENIZER_MAPPING._extra_content[CustomConfig]
										if CustomConfig in PROCESSOR_MAPPING._extra_content:
													del PROCESSOR_MAPPING._extra_content[CustomConfig]




				def 							A__      (						self					:						Optional[Any]						):
							lowercase__					  =				AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							self.assertEqual(processor.__class__.__name__,					"BertTokenizerFast"						)




				def 							A__      (						self					:						str						):
							lowercase__					  =				AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext"						)
							self.assertEqual(processor.__class__.__name__,					"ConvNextImageProcessor"						)
@is_staging_test
class 				_snake_case			(					unittest.TestCase):
				UpperCamelCase__				:					List[Any]     		=["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]




				@classmethod
				def 							A__      (						cls					:						Union[str, Any]						):
							lowercase__					  =				TOKEN
							HfFolder.save_token(__lowercase						)




				@classmethod
				def 							A__      (						cls					:						int						):
							try:
										delete_repo(token=cls._token,					repo_id="test-processor"						)
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token,					repo_id="valid_org/test-processor-org"						)
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token,					repo_id="test-dynamic-processor"						)
							except HTTPError:
										pass




				def 							A__      (						self					:						Dict						):
							lowercase__					  =				WavaVecaProcessor.from_pretrained(__lowercase						)
							with tempfile.TemporaryDirectory() as tmp_dir:
										processor.save_pretrained(
										    os.path.join(__lowercase,					"test-processor"						),					push_to_hub=__lowercase,					use_auth_token=self._token						)

										lowercase__					  =				WavaVecaProcessor.from_pretrained(F'''{USER}/test-processor'''						)
										for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(__lowercase,					getattr(new_processor.feature_extractor,					__lowercase						)						)
										self.assertDictEqual(new_processor.tokenizer.get_vocab(),					processor.tokenizer.get_vocab()						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				WavaVecaProcessor.from_pretrained(__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
										processor.save_pretrained(
										    os.path.join(__lowercase,					"test-processor-org"						),					push_to_hub=__lowercase,					use_auth_token=self._token,					organization="valid_org",					)

										lowercase__					  =				WavaVecaProcessor.from_pretrained("valid_org/test-processor-org"						)
										for k, v in processor.feature_extractor.__dict__.items():
													self.assertEqual(__lowercase,					getattr(new_processor.feature_extractor,					__lowercase						)						)
										self.assertDictEqual(new_processor.tokenizer.get_vocab(),					processor.tokenizer.get_vocab()						)




				def 							A__      (						self					:						List[Any]						):
							CustomFeatureExtractor.register_for_auto_class()
							CustomTokenizer.register_for_auto_class()
							CustomProcessor.register_for_auto_class()

							lowercase__					  =				CustomFeatureExtractor.from_pretrained(__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
										lowercase__					  =				os.path.join(__lowercase,					"vocab.txt"						)
										with open(__lowercase,					"w",					encoding="utf-8"						) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
										lowercase__					  =				CustomTokenizer(__lowercase						)

							lowercase__					  =				CustomProcessor(__lowercase,					__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
										create_repo(F'''{USER}/test-dynamic-processor''',					token=self._token						)
										lowercase__					  =				Repository(__lowercase,					clone_from=F'''{USER}/test-dynamic-processor''',					token=self._token						)
										processor.save_pretrained(__lowercase						)

										# This has added the proper auto_map field to the feature extractor config
										self.assertDictEqual(
										    processor.feature_extractor.auto_map,					{
										        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
										        "AutoProcessor": "custom_processing.CustomProcessor",
										    },					)

										# This has added the proper auto_map field to the tokenizer config
										with open(os.path.join(__lowercase,					"tokenizer_config.json"						)						) as f:
													lowercase__					  =				json.load(__lowercase						)
										self.assertDictEqual(
										    tokenizer_config["auto_map"],					{
										        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
										        "AutoProcessor": "custom_processing.CustomProcessor",
										    },					)

										# The code has been copied from fixtures
										self.assertTrue(os.path.isfile(os.path.join(__lowercase,					"custom_feature_extraction.py"						)						)						)
										self.assertTrue(os.path.isfile(os.path.join(__lowercase,					"custom_tokenization.py"						)						)						)
										self.assertTrue(os.path.isfile(os.path.join(__lowercase,					"custom_processing.py"						)						)						)

										repo.push_to_hub()

							lowercase__					  =				AutoProcessor.from_pretrained(F'''{USER}/test-dynamic-processor''',					trust_remote_code=__lowercase						)
							# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
							self.assertEqual(new_processor.__class__.__name__,					"CustomProcessor"						)









import re
from pathlib import Path
from unittest import TestCase

import pytest
@pytest.mark.integration
class 				_snake_case			(					lowercase__):




				def 							A__      (						self					:						Optional[Any],					__lowercase					:						str						):

							with open(__lowercase,					encoding="utf-8"						) as input_file:
										lowercase__					  =				re.compile(R"(?!.*\b(?:encoding|rb|w|wb|w+|wb+|ab|ab+)\b)(?<=\s)(open)\((.*)\)"						)
										lowercase__					  =				input_file.read()
										lowercase__					  =				regexp.search(__lowercase						)

							return match




				def 							A__      (						self					:						str,					__lowercase					:						str						):

							with open(__lowercase,					encoding="utf-8"						) as input_file:
										lowercase__					  =				re.compile(R"#[^\r\n]*print\(|\"[^\r\n]*print\(|\"\"\".*?print\(.*?\"\"\"|(print\()",					re.DOTALL						)
										lowercase__					  =				input_file.read()
										# use `re.finditer` to handle the case where the ignored groups would be matched first by `re.search`
										lowercase__					  =				regexp.finditer(__lowercase						)

							lowercase__					  =				[match for match in matches if match is not None and match.group(1						) is not None]
							return matches[0] if matches else None




				def 							A__      (						self					:						Union[str, Any]						):
							lowercase__					  =				Path("./datasets"						)
							lowercase__					  =				list(dataset_paths.absolute().glob("**/*.py"						)						)

							for dataset in dataset_files:
										if self._no_encoding_on_file_open(str(__lowercase						)						):
													raise AssertionError(F'''open(...) must use utf-8 encoding in {dataset}'''						)




				def 							A__      (						self					:						Union[str, Any]						):
							lowercase__					  =				Path("./datasets"						)
							lowercase__					  =				list(dataset_paths.absolute().glob("**/*.py"						)						)

							for dataset in dataset_files:
										if self._no_print_statements(str(__lowercase						)						):
													raise AssertionError(F'''print statement found in {dataset}. Use datasets.logger/logging instead.'''						)




