

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast

class 				a  (	unittest.TestCase				):


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    0


						@slow
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
														_UpperCAmelCase       :		Any 		=    AutoTokenizer.from_pretrained(A_	)
														self.assertIsNotNone(A_	)
														self.assertIsInstance(A_						,					(BertTokenizer, BertTokenizerFast)	)
														self.assertGreater(len(A_	)						,					0	)

										for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
														_UpperCAmelCase       :		Any 		=    AutoTokenizer.from_pretrained(A_	)
														self.assertIsNotNone(A_	)
														self.assertIsInstance(A_						,					(GPTaTokenizer, GPTaTokenizerFast)	)
														self.assertGreater(len(A_	)						,					0	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    AutoTokenizer.from_pretrained(A_	)
										self.assertIsInstance(A_						,					(BertTokenizer, BertTokenizerFast)	)
										self.assertEqual(tokenizer.vocab_size						,					12	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		str 		=    AutoTokenizer.from_pretrained(A_	)
										self.assertIsInstance(A_						,					(RobertaTokenizer, RobertaTokenizerFast)	)
										self.assertEqual(tokenizer.vocab_size						,					20	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    AutoConfig.from_pretrained(A_	)
										self.assertIsInstance(A_						,					A_	)
										# Check that tokenizer_type ≠ model_type
										_UpperCAmelCase       :		Dict 		=    AutoTokenizer.from_pretrained(A_						,					config=A_	)
										self.assertIsInstance(A_						,					(BertTokenizer, BertTokenizerFast)	)
										self.assertEqual(tokenizer.vocab_size						,					12	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy("./tests/fixtures/vocab.txt"						,					os.path.join(A_						,					"vocab.txt"	)	)

														_UpperCAmelCase       :		Tuple 		=    AutoTokenizer.from_pretrained(A_						,					tokenizer_type="bert"						,					use_fast=A_	)
														self.assertIsInstance(A_						,					A_	)

										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy("./tests/fixtures/vocab.json"						,					os.path.join(A_						,					"vocab.json"	)	)
														shutil.copy("./tests/fixtures/merges.txt"						,					os.path.join(A_						,					"merges.txt"	)	)

														_UpperCAmelCase       :		Union[str, Any] 		=    AutoTokenizer.from_pretrained(A_						,					tokenizer_type="gpt2"						,					use_fast=A_	)
														self.assertIsInstance(A_						,					A_	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy("./tests/fixtures/vocab.txt"						,					os.path.join(A_						,					"vocab.txt"	)	)

														_UpperCAmelCase       :		Optional[int] 		=    AutoTokenizer.from_pretrained(A_						,					tokenizer_type="bert"	)
														self.assertIsInstance(A_						,					A_	)

										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy("./tests/fixtures/vocab.json"						,					os.path.join(A_						,					"vocab.json"	)	)
														shutil.copy("./tests/fixtures/merges.txt"						,					os.path.join(A_						,					"merges.txt"	)	)

														_UpperCAmelCase       :		int 		=    AutoTokenizer.from_pretrained(A_						,					tokenizer_type="gpt2"	)
														self.assertIsInstance(A_						,					A_	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										with pytest.raises(A_	):
														AutoTokenizer.from_pretrained("./"						,					tokenizer_type="xxx"	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
														_UpperCAmelCase       :		List[str] 		=    tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"	)
														self.assertIsInstance(A_						,					(BertTokenizer, BertTokenizerFast)	)

														if isinstance(A_						,					A_	):
																		self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,					A_	)
														else:
																		self.assertEqual(tokenizer.do_lower_case						,					A_	)

														self.assertEqual(tokenizer.model_max_length						,					512	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
														with self.assertRaisesRegex(
														    A_						,					"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"						,					):
																		_UpperCAmelCase       :		List[str] 		=    tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    TOKENIZER_MAPPING.values()
										_UpperCAmelCase       :		Any 		=    []

										for slow_tok, fast_tok in tokenizers:
														if slow_tok is not None:
																		tokenizer_names.append(slow_tok.__name__	)

														if fast_tok is not None:
																		tokenizer_names.append(fast_tok.__name__	)

										for tokenizer_name in tokenizer_names:
														# must find the right class
														tokenizer_class_from_name(A_	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						,					use_fast=A_	)						,					A_	)
										self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"	)						,					A_	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[str] 		=    AutoTokenizer.from_pretrained("distilbert-base-uncased"						,					do_lower_case=A_	)
										_UpperCAmelCase       :		str 		=    "Hello, world. How are you?"
										_UpperCAmelCase       :		Optional[int] 		=    tokenizer.tokenize(A_	)
										self.assertEqual("[UNK]"						,					tokens[0]	)

										_UpperCAmelCase       :		Optional[Any] 		=    AutoTokenizer.from_pretrained("microsoft/mpnet-base"						,					do_lower_case=A_	)
										_UpperCAmelCase       :		Optional[int] 		=    tokenizer.tokenize(A_	)
										self.assertEqual("[UNK]"						,					tokens[0]	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		List[Any] 		=    AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"	)
										self.assertEqual(type(A_	)						,					A_	)
										self.assertEqual(tokenizer.model_max_length						,					512	)
										self.assertEqual(tokenizer.vocab_size						,					30000	)
										self.assertEqual(tokenizer.unk_token						,					"[UNK]"	)
										self.assertEqual(tokenizer.padding_side						,					"right"	)
										self.assertEqual(tokenizer.truncation_side						,					"right"	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    AutoTokenizer.from_pretrained(A_	)
										self.assertIsInstance(A_						,					(BertTokenizer, BertTokenizerFast)	)
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(A_	)
														_UpperCAmelCase       :		Tuple 		=    AutoTokenizer.from_pretrained(A_	)

										self.assertIsInstance(A_						,					tokenizer.__class__	)
										self.assertEqual(tokenizera.vocab_size						,					12	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    AutoTokenizer.from_pretrained("ctrl"	)
										# There is no fast CTRL so this always gives us a slow tokenizer.
										self.assertIsInstance(A_						,					A_	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    get_tokenizer_config("bert-base-cased"	)
										_UpperCAmelCase       :		Tuple 		=    config.pop("_commit_hash"						,					A_	)
										# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
										self.assertEqual(A_						,					{"do_lower_case": False}	)

										# This model does not have a tokenizer_config so we get back an empty dict.
										_UpperCAmelCase       :		Optional[Any] 		=    get_tokenizer_config(A_	)
										self.assertDictEqual(A_						,					{}	)

										# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
										_UpperCAmelCase       :		int 		=    AutoTokenizer.from_pretrained(A_	)
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(A_	)
														_UpperCAmelCase       :		Dict 		=    get_tokenizer_config(A_	)

										# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
										self.assertEqual(config["tokenizer_class"]						,					"BertTokenizer"	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										try:
														AutoConfig.register("custom"						,					A_	)

														AutoTokenizer.register(A_						,					slow_tokenizer_class=A_	)
														# Trying to register something existing in the Transformers library will raise an error
														with self.assertRaises(A_	):
																		AutoTokenizer.register(A_						,					slow_tokenizer_class=A_	)

														_UpperCAmelCase       :		Tuple 		=    CustomTokenizer.from_pretrained(A_	)
														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(A_	)

																		_UpperCAmelCase       :		Dict 		=    AutoTokenizer.from_pretrained(A_	)
																		self.assertIsInstance(A_						,					A_	)

										finally:
														if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
														if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										try:
														AutoConfig.register("custom"						,					A_	)

														# Can register in two steps
														AutoTokenizer.register(A_						,					slow_tokenizer_class=A_	)
														self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,					(CustomTokenizer, None)	)
														AutoTokenizer.register(A_						,					fast_tokenizer_class=A_	)
														self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,					(CustomTokenizer, CustomTokenizerFast)	)

														del TOKENIZER_MAPPING._extra_content[CustomConfig]
														# Can register in one step
														AutoTokenizer.register(
														    A_						,					slow_tokenizer_class=A_						,					fast_tokenizer_class=A_	)
														self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,					(CustomTokenizer, CustomTokenizerFast)	)

														# Trying to register something existing in the Transformers library will raise an error
														with self.assertRaises(A_	):
																		AutoTokenizer.register(A_						,					fast_tokenizer_class=A_	)

														# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
														# and that model does not have a tokenizer.json
														with tempfile.TemporaryDirectory() as tmp_dir:
																		_UpperCAmelCase       :		List[Any] 		=    BertTokenizerFast.from_pretrained(A_	)
																		bert_tokenizer.save_pretrained(A_	)
																		_UpperCAmelCase       :		Optional[int] 		=    CustomTokenizerFast.from_pretrained(A_	)

														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(A_	)

																		_UpperCAmelCase       :		Any 		=    AutoTokenizer.from_pretrained(A_	)
																		self.assertIsInstance(A_						,					A_	)

																		_UpperCAmelCase       :		Optional[Any] 		=    AutoTokenizer.from_pretrained(A_						,					use_fast=A_	)
																		self.assertIsInstance(A_						,					A_	)

										finally:
														if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
														if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										with self.assertRaises(A_	):
														_UpperCAmelCase       :		Union[str, Any] 		=    AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"	)
										# If remote code is disabled, we can't load this config.
										with self.assertRaises(A_	):
														_UpperCAmelCase       :		Optional[Any] 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_	)

										_UpperCAmelCase       :		Dict 		=    AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_	)
										self.assertTrue(tokenizer.special_attribute_present	)
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(A_	)
														_UpperCAmelCase       :		int 		=    AutoTokenizer.from_pretrained(A_						,					trust_remote_code=A_	)
										self.assertTrue(reloaded_tokenizer.special_attribute_present	)

										if is_tokenizers_available():
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizerFast"	)
														self.assertEqual(reloaded_tokenizer.__class__.__name__						,					"NewTokenizerFast"	)

														# Test we can also load the slow version
														_UpperCAmelCase       :		Dict 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_						,					use_fast=A_	)
														self.assertTrue(tokenizer.special_attribute_present	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)
														# Test tokenizer can be reloaded.
														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(A_	)
																		_UpperCAmelCase       :		Tuple 		=    AutoTokenizer.from_pretrained(A_						,					trust_remote_code=A_						,					use_fast=A_	)
														self.assertEqual(reloaded_tokenizer.__class__.__name__						,					"NewTokenizer"	)
														self.assertTrue(reloaded_tokenizer.special_attribute_present	)
										else:
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)
														self.assertEqual(reloaded_tokenizer.__class__.__name__						,					"NewTokenizer"	)


						@require_tokenizers
						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										class 				a  (	UpperCAmelCase				):
																_lowercase    						=			False







										class 				a  (	UpperCAmelCase				):
																_lowercase    						=			NewTokenizer
																_lowercase    						=			False

										try:
														AutoConfig.register("custom"						,					A_	)
														AutoTokenizer.register(A_						,					slow_tokenizer_class=A_	)
														AutoTokenizer.register(A_						,					fast_tokenizer_class=A_	)
														# If remote code is not set, the default is to use local
														_UpperCAmelCase       :		str 		=    AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizerFast"	)
														self.assertFalse(tokenizer.special_attribute_present	)
														_UpperCAmelCase       :		Optional[int] 		=    AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						,					use_fast=A_	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)
														self.assertFalse(tokenizer.special_attribute_present	)

														# If remote code is disabled, we load the local one.
														_UpperCAmelCase       :		int 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizerFast"	)
														self.assertFalse(tokenizer.special_attribute_present	)
														_UpperCAmelCase       :		List[Any] 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_						,					use_fast=A_	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)
														self.assertFalse(tokenizer.special_attribute_present	)

														# If remote is enabled, we load from the Hub
														_UpperCAmelCase       :		Optional[Any] 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizerFast"	)
														self.assertTrue(tokenizer.special_attribute_present	)
														_UpperCAmelCase       :		Optional[Any] 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer"						,					trust_remote_code=A_						,					use_fast=A_	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)
														self.assertTrue(tokenizer.special_attribute_present	)

										finally:
														if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
														if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    AutoTokenizer.from_pretrained(
										    "hf-internal-testing/test_dynamic_tokenizer_legacy"						,					trust_remote_code=A_	)
										self.assertTrue(tokenizer.special_attribute_present	)
										if is_tokenizers_available():
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizerFast"	)

														# Test we can also load the slow version
														_UpperCAmelCase       :		str 		=    AutoTokenizer.from_pretrained(
														    "hf-internal-testing/test_dynamic_tokenizer_legacy"						,					trust_remote_code=A_						,					use_fast=A_	)
														self.assertTrue(tokenizer.special_attribute_present	)
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)
										else:
														self.assertEqual(tokenizer.__class__.__name__						,					"NewTokenizer"	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										with self.assertRaisesRegex(
										    A_						,					"bert-base is not a local folder and is not a valid model identifier"	):
														_UpperCAmelCase       :		List[Any] 		=    AutoTokenizer.from_pretrained("bert-base"	)


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										with self.assertRaisesRegex(
										    A_						,					R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"	):
														_UpperCAmelCase       :		Optional[Any] 		=    AutoTokenizer.from_pretrained(A_						,					revision="aaaaaa"	)





						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Optional[int] 		=    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"	)
										with RequestCounter() as counter:
														_UpperCAmelCase       :		Dict 		=    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"	)
														self.assertEqual(counter.get_request_count						,					0	)
														self.assertEqual(counter.head_request_count						,					1	)
														self.assertEqual(counter.other_request_count						,					0	)





import logging
import os

from .state import PartialState

class 				a  (	logging.LoggerAdapter				):


						@staticmethod
						def        _UpperCAmelCase				(       A_	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    PartialState()
										return not main_process_only or (main_process_only and state.is_main_process)





						def        _UpperCAmelCase				(       self						,					A_						,					A_						,					*A_						,					**A_	):







										'''simple docstring'''


										if PartialState._shared_state == {}:
														raise RuntimeError(
														    "You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility."	)
										_UpperCAmelCase       :		Tuple 		=    kwargs.pop("main_process_only"						,					A_	)
										_UpperCAmelCase       :		int 		=    kwargs.pop("in_order"						,					A_	)

										if self.isEnabledFor(A_	):
														if self._should_log(A_	):
																		_UpperCAmelCase		,							_UpperCAmelCase       :		Optional[int] 		=    self.process(A_						,					A_	)
																		self.logger.log(A_						,					A_						,					*A_						,					**A_	)

														elif in_order:
																		_UpperCAmelCase       :		Dict 		=    PartialState()
																		for i in range(state.num_processes	):
																						if i == state.process_index:
																										_UpperCAmelCase		,							_UpperCAmelCase       :		Union[str, Any] 		=    self.process(A_						,					A_	)
																										self.logger.log(A_						,					A_						,					*A_						,					**A_	)
																						state.wait_for_everyone()







def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  str  ,      lowerCAmelCase:  str = None )   ->       List[Any]:
				if log_level is None:
								_UpperCAmelCase       :		List[str] 		=    os.environ.get("ACCELERATE_LOG_LEVEL"  ,      lowerCAmelCase )
				_UpperCAmelCase       :		str 		=    logging.getLogger(lowerCAmelCase )
				if log_level is not None:
								logger.setLevel(log_level.upper() )
								logger.root.setLevel(log_level.upper() )
				return MultiProcessAdapter(lowerCAmelCase  ,      {} )


