
import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
				from test_module.custom_tokenization_fast import CustomTokenizerFast






class        SCREAMING_SNAKE_CASE__    (							unittest.TestCase    ):

							"""simple docstring"""






							def     A__		(  self			)->   str:





													'''simple docstring'''






													__UpperCamelCase       =			0



							@slow
							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
																			self.assertIsNotNone(SCREAMING_SNAKE_CASE_			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(BertTokenizer, BertTokenizerFast)			)
																			self.assertGreater(len(SCREAMING_SNAKE_CASE_			)     ,	0			)

													for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
																			self.assertIsNotNone(SCREAMING_SNAKE_CASE_			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(GPTaTokenizer, GPTaTokenizerFast)			)
																			self.assertGreater(len(SCREAMING_SNAKE_CASE_			)     ,	0			)



							def     A__		(  self			)->   Dict:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(BertTokenizer, BertTokenizerFast)			)
													self.assertEqual(tokenizer.vocab_size     ,	12			)



							def     A__		(  self			)->   Dict:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(RobertaTokenizer, RobertaTokenizerFast)			)
													self.assertEqual(tokenizer.vocab_size     ,	20			)



							def     A__		(  self			)->   List[Any]:





													'''simple docstring'''






													__UpperCamelCase       =			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_			)
													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)
													# Check that tokenizer_type ≠ model_type
													__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	config=SCREAMING_SNAKE_CASE_			)
													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(BertTokenizer, BertTokenizerFast)			)
													self.assertEqual(tokenizer.vocab_size     ,	12			)



							def     A__		(  self			)->   str:





													'''simple docstring'''






													with tempfile.TemporaryDirectory() as tmp_dir:
																			shutil.copy('''./tests/fixtures/vocab.txt'''     ,	os.path.join(SCREAMING_SNAKE_CASE_     ,	'''vocab.txt'''			)			)

																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	tokenizer_type='''bert'''     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)

													with tempfile.TemporaryDirectory() as tmp_dir:
																			shutil.copy('''./tests/fixtures/vocab.json'''     ,	os.path.join(SCREAMING_SNAKE_CASE_     ,	'''vocab.json'''			)			)
																			shutil.copy('''./tests/fixtures/merges.txt'''     ,	os.path.join(SCREAMING_SNAKE_CASE_     ,	'''merges.txt'''			)			)

																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	tokenizer_type='''gpt2'''     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)



							@require_tokenizers
							def     A__		(  self			)->   Union[str, Any]:





													'''simple docstring'''






													with tempfile.TemporaryDirectory() as tmp_dir:
																			shutil.copy('''./tests/fixtures/vocab.txt'''     ,	os.path.join(SCREAMING_SNAKE_CASE_     ,	'''vocab.txt'''			)			)

																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	tokenizer_type='''bert'''			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)

													with tempfile.TemporaryDirectory() as tmp_dir:
																			shutil.copy('''./tests/fixtures/vocab.json'''     ,	os.path.join(SCREAMING_SNAKE_CASE_     ,	'''vocab.json'''			)			)
																			shutil.copy('''./tests/fixtures/merges.txt'''     ,	os.path.join(SCREAMING_SNAKE_CASE_     ,	'''merges.txt'''			)			)

																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	tokenizer_type='''gpt2'''			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)



							def     A__		(  self			)->   int:





													'''simple docstring'''






													with pytest.raises(SCREAMING_SNAKE_CASE_			):
																			AutoTokenizer.from_pretrained('''./'''     ,	tokenizer_type='''xxx'''			)



							@require_tokenizers
							def     A__		(  self			)->   Union[str, Any]:





													'''simple docstring'''






													for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
																			__UpperCamelCase       =			tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''			)
																			self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(BertTokenizer, BertTokenizerFast)			)

																			if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			):
																									self.assertEqual(tokenizer.basic_tokenizer.do_lower_case     ,	SCREAMING_SNAKE_CASE_			)
																			else:
																									self.assertEqual(tokenizer.do_lower_case     ,	SCREAMING_SNAKE_CASE_			)

																			self.assertEqual(tokenizer.model_max_length     ,	512			)



							@require_tokenizers
							def     A__		(  self			)->   int:





													'''simple docstring'''






													for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
																			with self.assertRaisesRegex(
																			    SCREAMING_SNAKE_CASE_     ,	'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''     ,	):
																									__UpperCamelCase       =			tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''			)



							def     A__		(  self			)->   Dict:





													'''simple docstring'''






													__UpperCamelCase       =			TOKENIZER_MAPPING.values()
													__UpperCamelCase       =			[]

													for slow_tok, fast_tok in tokenizers:
																			if slow_tok is not None:
																									tokenizer_names.append(slow_tok.__name__			)

																			if fast_tok is not None:
																									tokenizer_names.append(fast_tok.__name__			)

													for tokenizer_name in tokenizer_names:
																			# must find the right class
																			tokenizer_class_from_name(SCREAMING_SNAKE_CASE_			)



							@require_tokenizers
							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''     ,	use_fast=SCREAMING_SNAKE_CASE_			)     ,	SCREAMING_SNAKE_CASE_			)
													self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''			)     ,	SCREAMING_SNAKE_CASE_			)



							@require_tokenizers
							def     A__		(  self			)->   Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained('''distilbert-base-uncased'''     ,	do_lower_case=SCREAMING_SNAKE_CASE_			)
													__UpperCamelCase       =			'''Hello, world. How are you?'''
													__UpperCamelCase       =			tokenizer.tokenize(SCREAMING_SNAKE_CASE_			)
													self.assertEqual('''[UNK]'''     ,	tokens[0]			)

													__UpperCamelCase       =			AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''     ,	do_lower_case=SCREAMING_SNAKE_CASE_			)
													__UpperCamelCase       =			tokenizer.tokenize(SCREAMING_SNAKE_CASE_			)
													self.assertEqual('''[UNK]'''     ,	tokens[0]			)



							@require_tokenizers
							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''			)
													self.assertEqual(type(SCREAMING_SNAKE_CASE_			)     ,	SCREAMING_SNAKE_CASE_			)
													self.assertEqual(tokenizer.model_max_length     ,	512			)
													self.assertEqual(tokenizer.vocab_size     ,	30000			)
													self.assertEqual(tokenizer.unk_token     ,	'''[UNK]'''			)
													self.assertEqual(tokenizer.padding_side     ,	'''right'''			)
													self.assertEqual(tokenizer.truncation_side     ,	'''right'''			)



							def     A__		(  self			)->   Any:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	(BertTokenizer, BertTokenizerFast)			)
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)

													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	tokenizer.__class__			)
													self.assertEqual(tokenizera.vocab_size     ,	12			)



							def     A__		(  self			)->   Optional[int]:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained('''ctrl'''			)
													# There is no fast CTRL so this always gives us a slow tokenizer.
													self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)



							def     A__		(  self			)->   Union[str, Any]:





													'''simple docstring'''






													__UpperCamelCase       =			get_tokenizer_config('''bert-base-cased'''			)
													__UpperCamelCase       =			config.pop('''_commit_hash'''     ,	SCREAMING_SNAKE_CASE_			)
													# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
													self.assertEqual(SCREAMING_SNAKE_CASE_     ,	{'''do_lower_case''': False}			)

													# This model does not have a tokenizer_config so we get back an empty dict.
													__UpperCamelCase       =			get_tokenizer_config(SCREAMING_SNAKE_CASE_			)
													self.assertDictEqual(SCREAMING_SNAKE_CASE_     ,	{}			)

													# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
													__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
																			__UpperCamelCase       =			get_tokenizer_config(SCREAMING_SNAKE_CASE_			)

													# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
													self.assertEqual(config['''tokenizer_class''']     ,	'''BertTokenizer'''			)



							def     A__		(  self			)->   List[Any]:





													'''simple docstring'''






													try:
																			AutoConfig.register('''custom'''     ,	SCREAMING_SNAKE_CASE_			)

																			AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	slow_tokenizer_class=SCREAMING_SNAKE_CASE_			)
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(SCREAMING_SNAKE_CASE_			):
																									AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	slow_tokenizer_class=SCREAMING_SNAKE_CASE_			)

																			__UpperCamelCase       =			CustomTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
																			with tempfile.TemporaryDirectory() as tmp_dir:
																									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)

																									__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
																									self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]



							@require_tokenizers
							def     A__		(  self			)->   Union[str, Any]:





													'''simple docstring'''






													try:
																			AutoConfig.register('''custom'''     ,	SCREAMING_SNAKE_CASE_			)

																			# Can register in two steps
																			AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	slow_tokenizer_class=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(TOKENIZER_MAPPING[CustomConfig]     ,	(CustomTokenizer, None)			)
																			AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	fast_tokenizer_class=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(TOKENIZER_MAPPING[CustomConfig]     ,	(CustomTokenizer, CustomTokenizerFast)			)

																			del TOKENIZER_MAPPING._extra_content[CustomConfig]
																			# Can register in one step
																			AutoTokenizer.register(
																			    SCREAMING_SNAKE_CASE_     ,	slow_tokenizer_class=SCREAMING_SNAKE_CASE_     ,	fast_tokenizer_class=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(TOKENIZER_MAPPING[CustomConfig]     ,	(CustomTokenizer, CustomTokenizerFast)			)

																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(SCREAMING_SNAKE_CASE_			):
																									AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	fast_tokenizer_class=SCREAMING_SNAKE_CASE_			)

																			# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
																			# and that model does not have a tokenizer.json
																			with tempfile.TemporaryDirectory() as tmp_dir:
																									__UpperCamelCase       =			BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE_			)
																									bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
																									__UpperCamelCase       =			CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE_			)

																			with tempfile.TemporaryDirectory() as tmp_dir:
																									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)

																									__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
																									self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)

																									__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																									self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]



							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													with self.assertRaises(SCREAMING_SNAKE_CASE_			):
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''			)
													# If remote code is disabled, we can't load this config.
													with self.assertRaises(SCREAMING_SNAKE_CASE_			):
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_			)

													__UpperCamelCase       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_			)
													self.assertTrue(tokenizer.special_attribute_present			)
													# Test tokenizer can be reloaded.
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	trust_remote_code=SCREAMING_SNAKE_CASE_			)
													self.assertTrue(reloaded_tokenizer.special_attribute_present			)

													if is_tokenizers_available():
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizerFast'''			)
																			self.assertEqual(reloaded_tokenizer.__class__.__name__     ,	'''NewTokenizerFast'''			)

																			# Test we can also load the slow version
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertTrue(tokenizer.special_attribute_present			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
																			# Test tokenizer can be reloaded.
																			with tempfile.TemporaryDirectory() as tmp_dir:
																									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
																									__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	trust_remote_code=SCREAMING_SNAKE_CASE_     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(reloaded_tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
																			self.assertTrue(reloaded_tokenizer.special_attribute_present			)
													else:
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
																			self.assertEqual(reloaded_tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)



							@require_tokenizers
							def     A__		(  self			)->   int:





													'''simple docstring'''






													class        SCREAMING_SNAKE_CASE__    (							SCREAMING_SNAKE_CASE_    ):

																				"""simple docstring"""






																				_snake_case     		=  False






													class        SCREAMING_SNAKE_CASE__    (							SCREAMING_SNAKE_CASE_    ):

																				"""simple docstring"""






																				_snake_case     		=  NewTokenizer
																				_snake_case     		=  False

													try:
																			AutoConfig.register('''custom'''     ,	SCREAMING_SNAKE_CASE_			)
																			AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	slow_tokenizer_class=SCREAMING_SNAKE_CASE_			)
																			AutoTokenizer.register(SCREAMING_SNAKE_CASE_     ,	fast_tokenizer_class=SCREAMING_SNAKE_CASE_			)
																			# If remote code is not set, the default is to use local
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizerFast'''			)
																			self.assertFalse(tokenizer.special_attribute_present			)
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
																			self.assertFalse(tokenizer.special_attribute_present			)

																			# If remote code is disabled, we load the local one.
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizerFast'''			)
																			self.assertFalse(tokenizer.special_attribute_present			)
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
																			self.assertFalse(tokenizer.special_attribute_present			)

																			# If remote is enabled, we load from the Hub
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizerFast'''			)
																			self.assertTrue(tokenizer.special_attribute_present			)
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
																			self.assertTrue(tokenizer.special_attribute_present			)

													finally:
																			if "custom" in CONFIG_MAPPING._extra_content:
																									del CONFIG_MAPPING._extra_content["custom"]
																			if CustomConfig in TOKENIZER_MAPPING._extra_content:
																									del TOKENIZER_MAPPING._extra_content[CustomConfig]



							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained(
													    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_			)
													self.assertTrue(tokenizer.special_attribute_present			)
													if is_tokenizers_available():
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizerFast'''			)

																			# Test we can also load the slow version
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(
																			    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''     ,	trust_remote_code=SCREAMING_SNAKE_CASE_     ,	use_fast=SCREAMING_SNAKE_CASE_			)
																			self.assertTrue(tokenizer.special_attribute_present			)
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)
													else:
																			self.assertEqual(tokenizer.__class__.__name__     ,	'''NewTokenizer'''			)



							def     A__		(  self			)->   Union[str, Any]:





													'''simple docstring'''






													with self.assertRaisesRegex(
													    SCREAMING_SNAKE_CASE_     ,	'''bert-base is not a local folder and is not a valid model identifier'''			):
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained('''bert-base'''			)



							def     A__		(  self			)->   Tuple:





													'''simple docstring'''






													with self.assertRaisesRegex(
													    SCREAMING_SNAKE_CASE_     ,	r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''			):
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_     ,	revision='''aaaaaa'''			)



							def     A__		(  self			)->   Any:





													'''simple docstring'''






													__UpperCamelCase       =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
													with RequestCounter() as counter:
																			__UpperCamelCase       =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
																			self.assertEqual(counter.get_request_count     ,	0			)
																			self.assertEqual(counter.head_request_count     ,	1			)
																			self.assertEqual(counter.other_request_count     ,	0			)






# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


lowercase__							:							List[str]									=			"3"

print("Python version:", sys.version)

print("OS platform:", platform.platform())
print("OS architecture:", platform.machine())

try:
				import torch

				print("Torch version:", torch.__version__)
				print("Cuda available:", torch.cuda.is_available())
				print("Cuda version:", torch.version.cuda)
				print("CuDNN version:", torch.backends.cudnn.version())
				print("Number of GPUs available:", torch.cuda.device_count())
except ImportError:
				print("Torch version:", None)

try:
				import transformers

				print("transformers version:", transformers.__version__)
except ImportError:
				print("transformers version:", None)




