





import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


__lowercase       :Any       =  datasets.utils.logging.get_logger(__name__)

__lowercase       :Tuple       =  ["names", "prefix"]
__lowercase       :str       =  ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"]
__lowercase       :List[Any]       =  ["encoding_errors", "on_bad_lines"]
__lowercase       :List[Any]       =  ["date_format"]



@dataclass
class    _a     (				datasets.BuilderConfig							):






							"""simple docstring"""

							snake_case_							 =						","
							snake_case_							 =						None
							snake_case_							 =						"infer"
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						True
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						False
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						True
							snake_case_							 =						True
							snake_case_							 =						False
							snake_case_							 =						True
							snake_case_							 =						None
							snake_case_							 =						"."
							snake_case_							 =						None
							snake_case_							 =						'"'
							snake_case_							 =						0
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						None
							snake_case_							 =						True
							snake_case_							 =						True
							snake_case_							 =						0
							snake_case_							 =						True
							snake_case_							 =						False
							snake_case_							 =						None
							snake_case_							 =						1_00_00
							snake_case_							 =						None
							snake_case_							 =						"strict"
							snake_case_							 =						"error"
							snake_case_							 =						None

							def      A_  (      self				:     Optional[int]    )	->int:
													if self.delimiter is not None:
																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					self.delimiter
													if self.column_names is not None:
																			SCREAMING_SNAKE_CASE__     :				int   =					self.column_names






							@property
							def      A_  (      self				:     List[Any]    )	->Any:
													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					{
													    "sep": self.sep,
													    "header": self.header,
													    "names": self.names,
													    "index_col": self.index_col,
													    "usecols": self.usecols,
													    "prefix": self.prefix,
													    "mangle_dupe_cols": self.mangle_dupe_cols,
													    "engine": self.engine,
													    "converters": self.converters,
													    "true_values": self.true_values,
													    "false_values": self.false_values,
													    "skipinitialspace": self.skipinitialspace,
													    "skiprows": self.skiprows,
													    "nrows": self.nrows,
													    "na_values": self.na_values,
													    "keep_default_na": self.keep_default_na,
													    "na_filter": self.na_filter,
													    "verbose": self.verbose,
													    "skip_blank_lines": self.skip_blank_lines,
													    "thousands": self.thousands,
													    "decimal": self.decimal,
													    "lineterminator": self.lineterminator,
													    "quotechar": self.quotechar,
													    "quoting": self.quoting,
													    "escapechar": self.escapechar,
													    "comment": self.comment,
													    "encoding": self.encoding,
													    "dialect": self.dialect,
													    "error_bad_lines": self.error_bad_lines,
													    "warn_bad_lines": self.warn_bad_lines,
													    "skipfooter": self.skipfooter,
													    "doublequote": self.doublequote,
													    "memory_map": self.memory_map,
													    "float_precision": self.float_precision,
													    "chunksize": self.chunksize,
													    "encoding_errors": self.encoding_errors,
													    "on_bad_lines": self.on_bad_lines,
													    "date_format": self.date_format,
													}

													# some kwargs must not be passed if they don't have a default value
													# some others are deprecated and we can also not pass them if they are the default value
													for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
																			if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()							,	a    ):
																									del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
													if not (datasets.config.PANDAS_VERSION.major >= 2):
																			for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
																									del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
													if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
																			for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
																									del pd_read_csv_kwargs[pd_read_csv_parameter]

													return pd_read_csv_kwargs







class    _a     (				datasets.ArrowBasedBuilder							):






							"""simple docstring"""

							snake_case_							 =						CsvConfig

							def      A_  (      self				:     Union[str, Any]    )	->int:
													return datasets.DatasetInfo(features=self.config.features    )

							def      A_  (      self				:     Any							,	a				:     List[str]    )	->int:
													if not self.config.data_files:
																			raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					dl_manager.download_and_extract(self.config.data_files    )
													if isinstance(a							,	(str, list, tuple)    ):
																			SCREAMING_SNAKE_CASE__     :				List[str]   =					data_files
																			if isinstance(a							,	a    ):
																									SCREAMING_SNAKE_CASE__     :				str   =					[files]
																			SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					[dl_manager.iter_files(a    ) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,	gen_kwargs={"files": files}    )]
													SCREAMING_SNAKE_CASE__     :				str   =					[]
													for split_name, files in data_files.items():
																			if isinstance(a							,	a    ):
																									SCREAMING_SNAKE_CASE__     :				Any   =					[files]
																			SCREAMING_SNAKE_CASE__     :				int   =					[dl_manager.iter_files(a    ) for file in files]
																			splits.append(datasets.SplitGenerator(name=a							,	gen_kwargs={"files": files}    )    )
													return splits

							def      A_  (      self				:     Optional[Any]							,	a				:     pa.Table    )	->pa.Table:
													if self.config.features is not None:
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					self.config.features.arrow_schema
																			if all(not require_storage_cast(a    ) for feature in self.config.features.values()    ):
																									# cheaper cast
																									SCREAMING_SNAKE_CASE__     :				int   =					pa.Table.from_arrays([pa_table[field.name] for field in schema]							,	schema=a    )
																			else:
																									# more expensive cast; allows str <-> int/float or str to Audio for example
																									SCREAMING_SNAKE_CASE__     :				int   =					table_cast(a							,	a    )
													return pa_table






							def      A_  (      self				:     List[Any]							,	a				:     str    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				List[str]   =					self.config.features.arrow_schema if self.config.features else None
													# dtype allows reading an int column as str
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					(
													    {
													        name: dtype.to_pandas_dtype() if not require_storage_cast(a    ) else object
													        for name, dtype, feature in zip(schema.names							,	schema.types							,	self.config.features.values()    )
													    }
													    if schema is not None
													    else None
													)
													for file_idx, file in enumerate(itertools.chain.from_iterable(a    )    ):
																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					pd.read_csv(a							,	iterator=a							,	dtype=a							,	**self.config.pd_read_csv_kwargs    )
																			try:
																									for batch_idx, df in enumerate(a    ):
																															SCREAMING_SNAKE_CASE__     :				List[str]   =					pa.Table.from_pandas(a    )
																															# Uncomment for debugging (will print the Arrow table size and elements)
																															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield (file_idx, batch_idx), self._cast_table(a    )
																			except ValueError as e:
																									logger.error(f"""Failed to read file '{file}' with error {type(a    )}: {e}"""    )
																									raise






import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


__lowercase       :List[str]       =  get_tests_dir("fixtures/dummy_feature_extractor_config.json")
__lowercase       :str       =  get_tests_dir("fixtures/vocab.json")
__lowercase       :Optional[int]       =  get_tests_dir("fixtures")



class    _a     (				unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

							def      A_  (      self				:     Optional[Any]    )	->int:
													SCREAMING_SNAKE_CASE__     :				Dict   =					0

							def      A_  (      self				:     Any    )	->Optional[int]:
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"    )
													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     Union[str, Any]    )	->List[str]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				Dict   =					WavaVecaConfig()
																			SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"    )

																			# save in new folder
																			model_config.save_pretrained(a    )
																			processor.save_pretrained(a    )

																			SCREAMING_SNAKE_CASE__     :				str   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     int    )	->List[str]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			# copy relevant files
																			copyfile(a							,	os.path.join(a							,	a    )    )
																			copyfile(a							,	os.path.join(a							,	"vocab.json"    )    )

																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     List[Any]    )	->Tuple:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					WavaVecaFeatureExtractor()
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"    )

																			SCREAMING_SNAKE_CASE__     :				Any   =					WavaVecaProcessor(a							,	a    )

																			# save in new folder
																			processor.save_pretrained(a    )

																			# drop `processor_class` in tokenizer
																			with open(os.path.join(a							,	a    )							,	"r"    ) as f:
																									SCREAMING_SNAKE_CASE__     :				Optional[int]   =					json.load(a    )
																									config_dict.pop("processor_class"    )

																			with open(os.path.join(a							,	a    )							,	"w"    ) as f:
																									f.write(json.dumps(a    )    )

																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     List[str]    )	->Optional[Any]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					WavaVecaFeatureExtractor()
																			SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"    )

																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					WavaVecaProcessor(a							,	a    )

																			# save in new folder
																			processor.save_pretrained(a    )

																			# drop `processor_class` in feature extractor
																			with open(os.path.join(a							,	a    )							,	"r"    ) as f:
																									SCREAMING_SNAKE_CASE__     :				List[Any]   =					json.load(a    )
																									config_dict.pop("processor_class"    )

																			with open(os.path.join(a							,	a    )							,	"w"    ) as f:
																									f.write(json.dumps(a    )    )

																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     Union[str, Any]    )	->str:
													with tempfile.TemporaryDirectory() as tmpdirname:
																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					WavaVecaConfig(processor_class="Wav2Vec2Processor"    )
																			model_config.save_pretrained(a    )
																			# copy relevant files
																			copyfile(a							,	os.path.join(a							,	"vocab.json"    )    )
																			# create emtpy sample processor
																			with open(os.path.join(a							,	a    )							,	"w"    ) as f:
																									f.write("{}"    )

																			SCREAMING_SNAKE_CASE__     :				Tuple   =					AutoProcessor.from_pretrained(a    )

													self.assertIsInstance(a							,	a    )

							def      A_  (      self				:     Optional[Any]    )	->Optional[int]:
													# If remote code is not set, we will time out when asking whether to load the model.
													with self.assertRaises(a    ):
																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"    )
													# If remote code is disabled, we can't load this config.
													with self.assertRaises(a    ):
																			SCREAMING_SNAKE_CASE__     :				Any   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )

													SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )
													self.assertTrue(processor.special_attribute_present    )
													self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )

													SCREAMING_SNAKE_CASE__     :				Dict   =					processor.feature_extractor
													self.assertTrue(feature_extractor.special_attribute_present    )
													self.assertEqual(feature_extractor.__class__.__name__							,	"NewFeatureExtractor"    )

													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					processor.tokenizer
													self.assertTrue(tokenizer.special_attribute_present    )
													if is_tokenizers_available():
																			self.assertEqual(tokenizer.__class__.__name__							,	"NewTokenizerFast"    )

																			# Test we can also load the slow version
																			SCREAMING_SNAKE_CASE__     :				int   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a							,	use_fast=a    )
																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					new_processor.tokenizer
																			self.assertTrue(new_tokenizer.special_attribute_present    )
																			self.assertEqual(new_tokenizer.__class__.__name__							,	"NewTokenizer"    )
													else:
																			self.assertEqual(tokenizer.__class__.__name__							,	"NewTokenizer"    )

							def      A_  (      self				:     Tuple    )	->List[Any]:
													try:
																			AutoConfig.register("custom"							,	a    )
																			AutoFeatureExtractor.register(a							,	a    )
																			AutoTokenizer.register(a							,	slow_tokenizer_class=a    )
																			AutoProcessor.register(a							,	a    )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(a    ):
																									AutoProcessor.register(a							,	a    )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			SCREAMING_SNAKE_CASE__     :				List[str]   =					CustomFeatureExtractor.from_pretrained(a    )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																									SCREAMING_SNAKE_CASE__     :				int   =					os.path.join(a							,	"vocab.txt"    )
																									with open(a							,	"w"							,	encoding="utf-8"    ) as vocab_writer:
																															vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
																									SCREAMING_SNAKE_CASE__     :				Optional[int]   =					CustomTokenizer(a    )

																			SCREAMING_SNAKE_CASE__     :				List[Any]   =					CustomProcessor(a							,	a    )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																									processor.save_pretrained(a    )
																									SCREAMING_SNAKE_CASE__     :				Any   =					AutoProcessor.from_pretrained(a    )
																									self.assertIsInstance(a							,	a    )

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in PROCESSOR_MAPPING._extra_content:
																									del PROCESSOR_MAPPING._extra_content[CustomConfig]

							def      A_  (      self				:     Union[str, Any]    )	->int:



													class    _a     (				lowercase__							):






																				"""simple docstring"""

																				snake_case_							 =						False







													class    _a     (				lowercase__							):






																				"""simple docstring"""

																				snake_case_							 =						False







													class    _a     (				lowercase__							):






																				"""simple docstring"""

																				snake_case_							 =						"AutoFeatureExtractor"
																				snake_case_							 =						"AutoTokenizer"
																				snake_case_							 =						False

													try:
																			AutoConfig.register("custom"							,	a    )
																			AutoFeatureExtractor.register(a							,	a    )
																			AutoTokenizer.register(a							,	slow_tokenizer_class=a    )
																			AutoProcessor.register(a							,	a    )
																			# If remote code is not set, the default is to use local classes.
																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"    )
																			self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )
																			self.assertFalse(processor.special_attribute_present    )
																			self.assertFalse(processor.feature_extractor.special_attribute_present    )
																			self.assertFalse(processor.tokenizer.special_attribute_present    )

																			# If remote code is disabled, we load the local ones.
																			SCREAMING_SNAKE_CASE__     :				Tuple   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )
																			self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )
																			self.assertFalse(processor.special_attribute_present    )
																			self.assertFalse(processor.feature_extractor.special_attribute_present    )
																			self.assertFalse(processor.tokenizer.special_attribute_present    )

																			# If remote is enabled, we load from the Hub.
																			SCREAMING_SNAKE_CASE__     :				Any   =					AutoProcessor.from_pretrained(
																			    "hf-internal-testing/test_dynamic_processor"							,	trust_remote_code=a    )
																			self.assertEqual(processor.__class__.__name__							,	"NewProcessor"    )
																			self.assertTrue(processor.special_attribute_present    )
																			self.assertTrue(processor.feature_extractor.special_attribute_present    )
																			self.assertTrue(processor.tokenizer.special_attribute_present    )

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]
																			if CustomConfig in PROCESSOR_MAPPING._extra_content:
																									del PROCESSOR_MAPPING._extra_content[CustomConfig]

							def      A_  (      self				:     Optional[Any]    )	->Dict:
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert"    )
													self.assertEqual(processor.__class__.__name__							,	"BertTokenizerFast"    )






							def      A_  (      self				:     Dict    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				Dict   =					AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext"    )
													self.assertEqual(processor.__class__.__name__							,	"ConvNextImageProcessor"    )







@is_staging_test
class    _a     (				unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

							@classmethod
							def      A_  (      cls				:     List[str]    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				int   =					TOKEN
													HfFolder.save_token(a    )

							@classmethod
							def      A_  (      cls				:     List[str]    )	->Optional[int]:
													try:
																			delete_repo(token=cls._token							,	repo_id="test-processor"    )
													except HTTPError:
																			pass

													try:
																			delete_repo(token=cls._token							,	repo_id="valid_org/test-processor-org"    )
													except HTTPError:
																			pass

													try:
																			delete_repo(token=cls._token							,	repo_id="test-dynamic-processor"    )
													except HTTPError:
																			pass

							def      A_  (      self				:     Dict    )	->Dict:
													SCREAMING_SNAKE_CASE__     :				Tuple   =					WavaVecaProcessor.from_pretrained(a    )
													with tempfile.TemporaryDirectory() as tmp_dir:
																			processor.save_pretrained(
																			    os.path.join(a							,	"test-processor"    )							,	push_to_hub=a							,	use_auth_token=self._token    )

																			SCREAMING_SNAKE_CASE__     :				Optional[int]   =					WavaVecaProcessor.from_pretrained(f"""{USER}/test-processor"""    )
																			for k, v in processor.feature_extractor.__dict__.items():
																									self.assertEqual(a							,	getattr(new_processor.feature_extractor							,	a    )    )
																			self.assertDictEqual(new_processor.tokenizer.get_vocab()							,	processor.tokenizer.get_vocab()    )

							def      A_  (      self				:     List[str]    )	->Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					WavaVecaProcessor.from_pretrained(a    )

													with tempfile.TemporaryDirectory() as tmp_dir:
																			processor.save_pretrained(
																			    os.path.join(a							,	"test-processor-org"    )							,	push_to_hub=a							,	use_auth_token=self._token							,	organization="valid_org"							,	)

																			SCREAMING_SNAKE_CASE__     :				Dict   =					WavaVecaProcessor.from_pretrained("valid_org/test-processor-org"    )
																			for k, v in processor.feature_extractor.__dict__.items():
																									self.assertEqual(a							,	getattr(new_processor.feature_extractor							,	a    )    )
																			self.assertDictEqual(new_processor.tokenizer.get_vocab()							,	processor.tokenizer.get_vocab()    )






							def      A_  (      self				:     Any    )	->int:
													CustomFeatureExtractor.register_for_auto_class()
													CustomTokenizer.register_for_auto_class()
													CustomProcessor.register_for_auto_class()

													SCREAMING_SNAKE_CASE__     :				Any   =					CustomFeatureExtractor.from_pretrained(a    )

													with tempfile.TemporaryDirectory() as tmp_dir:
																			SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					os.path.join(a							,	"vocab.txt"    )
																			with open(a							,	"w"							,	encoding="utf-8"    ) as vocab_writer:
																									vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
																			SCREAMING_SNAKE_CASE__     :				str   =					CustomTokenizer(a    )

													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					CustomProcessor(a							,	a    )

													with tempfile.TemporaryDirectory() as tmp_dir:
																			create_repo(f"""{USER}/test-dynamic-processor"""							,	token=self._token    )
																			SCREAMING_SNAKE_CASE__     :				str   =					Repository(a							,	clone_from=f"""{USER}/test-dynamic-processor"""							,	token=self._token    )
																			processor.save_pretrained(a    )

																			# This has added the proper auto_map field to the feature extractor config
																			self.assertDictEqual(
																			    processor.feature_extractor.auto_map							,	{
																			        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
																			        "AutoProcessor": "custom_processing.CustomProcessor",
																			    }							,	)

																			# This has added the proper auto_map field to the tokenizer config
																			with open(os.path.join(a							,	"tokenizer_config.json"    )    ) as f:
																									SCREAMING_SNAKE_CASE__     :				str   =					json.load(a    )
																			self.assertDictEqual(
																			    tokenizer_config["auto_map"]							,	{
																			        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
																			        "AutoProcessor": "custom_processing.CustomProcessor",
																			    }							,	)

																			# The code has been copied from fixtures
																			self.assertTrue(os.path.isfile(os.path.join(a							,	"custom_feature_extraction.py"    )    )    )
																			self.assertTrue(os.path.isfile(os.path.join(a							,	"custom_tokenization.py"    )    )    )
																			self.assertTrue(os.path.isfile(os.path.join(a							,	"custom_processing.py"    )    )    )

																			repo.push_to_hub()

													SCREAMING_SNAKE_CASE__     :				List[Any]   =					AutoProcessor.from_pretrained(f"""{USER}/test-dynamic-processor"""							,	trust_remote_code=a    )
													# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
													self.assertEqual(new_processor.__class__.__name__							,	"CustomProcessor"    )