




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast




class    __lowerCAmelCase       (							unittest.TestCase):






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         lowercase    :Tuple      	=						0






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
           lowercase    :Dict      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
           self.assertIsNotNone(_lowerCAmelCase				)
           self.assertIsInstance(_lowerCAmelCase       ,					(BertTokenizer, BertTokenizerFast)				)
           self.assertGreater(len(_lowerCAmelCase				)       ,					0				)

         for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
           lowercase    :int      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
           self.assertIsNotNone(_lowerCAmelCase				)
           self.assertIsInstance(_lowerCAmelCase       ,					(GPTaTokenizer, GPTaTokenizerFast)				)
           self.assertGreater(len(_lowerCAmelCase				)       ,					0				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         lowercase    :List[Any]      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
         self.assertIsInstance(_lowerCAmelCase       ,					(BertTokenizer, BertTokenizerFast)				)
         self.assertEqual(tokenizer.vocab_size       ,					12				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :int      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
         self.assertIsInstance(_lowerCAmelCase       ,					(RobertaTokenizer, RobertaTokenizerFast)				)
         self.assertEqual(tokenizer.vocab_size       ,					20				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         lowercase    :Dict      	=						AutoConfig.from_pretrained(_lowerCAmelCase				)
         self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)
         # Check that tokenizer_type ≠ model_type
         lowercase    :Tuple      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					config=_lowerCAmelCase				)
         self.assertIsInstance(_lowerCAmelCase       ,					(BertTokenizer, BertTokenizerFast)				)
         self.assertEqual(tokenizer.vocab_size       ,					12				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.txt"       ,					os.path.join(_lowerCAmelCase       ,					"vocab.txt"				)				)

           lowercase    :Tuple      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					tokenizer_type="bert"       ,					use_fast=_lowerCAmelCase				)
           self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)

         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.json"       ,					os.path.join(_lowerCAmelCase       ,					"vocab.json"				)				)
           shutil.copy("./tests/fixtures/merges.txt"       ,					os.path.join(_lowerCAmelCase       ,					"merges.txt"				)				)

           lowercase    :str      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					tokenizer_type="gpt2"       ,					use_fast=_lowerCAmelCase				)
           self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.txt"       ,					os.path.join(_lowerCAmelCase       ,					"vocab.txt"				)				)

           lowercase    :int      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					tokenizer_type="bert"				)
           self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)

         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.json"       ,					os.path.join(_lowerCAmelCase       ,					"vocab.json"				)				)
           shutil.copy("./tests/fixtures/merges.txt"       ,					os.path.join(_lowerCAmelCase       ,					"merges.txt"				)				)

           lowercase    :Tuple      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					tokenizer_type="gpt2"				)
           self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         with pytest.raises(_lowerCAmelCase				):
           AutoTokenizer.from_pretrained("./"       ,					tokenizer_type="xxx"				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           lowercase    :Tuple      	=						tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"				)
           self.assertIsInstance(_lowerCAmelCase       ,					(BertTokenizer, BertTokenizerFast)				)

           if isinstance(_lowerCAmelCase       ,					_lowerCAmelCase				):
             self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,					_lowerCAmelCase				)
           else:
             self.assertEqual(tokenizer.do_lower_case       ,					_lowerCAmelCase				)

           self.assertEqual(tokenizer.model_max_length       ,					5_12				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           with self.assertRaisesRegex(
               _lowerCAmelCase       ,					"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"       ,					):
             lowercase    :Dict      	=						tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         # tests: https://github.com/huggingface/transformers/pull/13251
         # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
         # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
         lowercase    :Union[str, Any]      	=						TOKENIZER_MAPPING.values()
         lowercase    :Optional[int]      	=						[]

         for slow_tok, fast_tok in tokenizers:
           if slow_tok is not None:
             tokenizer_names.append(slow_tok.__name__				)

           if fast_tok is not None:
             tokenizer_names.append(fast_tok.__name__				)

         for tokenizer_name in tokenizer_names:
           # must find the right class
           tokenizer_class_from_name(_lowerCAmelCase				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"       ,					use_fast=_lowerCAmelCase				)       ,					_lowerCAmelCase				)
         self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"				)       ,					_lowerCAmelCase				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :Union[str, Any]      	=						AutoTokenizer.from_pretrained("distilbert-base-uncased"       ,					do_lower_case=_lowerCAmelCase				)
         lowercase    :str      	=						"Hello, world. How are you?"
         lowercase    :List[str]      	=						tokenizer.tokenize(_lowerCAmelCase				)
         self.assertEqual("[UNK]"       ,					tokens[0]				)

         lowercase    :str      	=						AutoTokenizer.from_pretrained("microsoft/mpnet-base"       ,					do_lower_case=_lowerCAmelCase				)
         lowercase    :Optional[int]      	=						tokenizer.tokenize(_lowerCAmelCase				)
         self.assertEqual("[UNK]"       ,					tokens[0]				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         lowercase    :Optional[Any]      	=						AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"				)
         self.assertEqual(type(_lowerCAmelCase				)       ,					_lowerCAmelCase				)
         self.assertEqual(tokenizer.model_max_length       ,					5_12				)
         self.assertEqual(tokenizer.vocab_size       ,					3_00_00				)
         self.assertEqual(tokenizer.unk_token       ,					"[UNK]"				)
         self.assertEqual(tokenizer.padding_side       ,					"right"				)
         self.assertEqual(tokenizer.truncation_side       ,					"right"				)






       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         lowercase    :int      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
         self.assertIsInstance(_lowerCAmelCase       ,					(BertTokenizer, BertTokenizerFast)				)
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCAmelCase				)
           lowercase    :Union[str, Any]      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)

         self.assertIsInstance(_lowerCAmelCase       ,					tokenizer.__class__				)
         self.assertEqual(tokenizera.vocab_size       ,					12				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         lowercase    :Optional[Any]      	=						AutoTokenizer.from_pretrained("ctrl"				)
         # There is no fast CTRL so this always gives us a slow tokenizer.
         self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         # Check we can load the tokenizer config of an online model.
         lowercase    :str      	=						get_tokenizer_config("bert-base-cased"				)
         lowercase    :Tuple      	=						config.pop("_commit_hash"       ,					_lowerCAmelCase				)
         # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
         self.assertEqual(_lowerCAmelCase       ,					{"do_lower_case": False}				)

         # This model does not have a tokenizer_config so we get back an empty dict.
         lowercase    :List[str]      	=						get_tokenizer_config(_lowerCAmelCase				)
         self.assertDictEqual(_lowerCAmelCase       ,					{}				)

         # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
         lowercase    :str      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCAmelCase				)
           lowercase    :Tuple      	=						get_tokenizer_config(_lowerCAmelCase				)

         # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
         self.assertEqual(config["tokenizer_class"]       ,					"BertTokenizer"				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         try:
           AutoConfig.register("custom"       ,					_lowerCAmelCase				)

           AutoTokenizer.register(_lowerCAmelCase       ,					slow_tokenizer_class=_lowerCAmelCase				)
           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(_lowerCAmelCase				):
             AutoTokenizer.register(_lowerCAmelCase       ,					slow_tokenizer_class=_lowerCAmelCase				)

           lowercase    :Tuple      	=						CustomTokenizer.from_pretrained(_lowerCAmelCase				)
           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(_lowerCAmelCase				)

             lowercase    :Any      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
             self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)

         finally:
           if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         try:
           AutoConfig.register("custom"       ,					_lowerCAmelCase				)

           # Can register in two steps
           AutoTokenizer.register(_lowerCAmelCase       ,					slow_tokenizer_class=_lowerCAmelCase				)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,					(CustomTokenizer, None)				)
           AutoTokenizer.register(_lowerCAmelCase       ,					fast_tokenizer_class=_lowerCAmelCase				)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,					(CustomTokenizer, CustomTokenizerFast)				)

           del TOKENIZER_MAPPING._extra_content[CustomConfig]
           # Can register in one step
           AutoTokenizer.register(
               _lowerCAmelCase       ,					slow_tokenizer_class=_lowerCAmelCase       ,					fast_tokenizer_class=_lowerCAmelCase				)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,					(CustomTokenizer, CustomTokenizerFast)				)

           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(_lowerCAmelCase				):
             AutoTokenizer.register(_lowerCAmelCase       ,					fast_tokenizer_class=_lowerCAmelCase				)

           # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
           # and that model does not have a tokenizer.json
           with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase    :List[str]      	=						BertTokenizerFast.from_pretrained(_lowerCAmelCase				)
             bert_tokenizer.save_pretrained(_lowerCAmelCase				)
             lowercase    :Union[str, Any]      	=						CustomTokenizerFast.from_pretrained(_lowerCAmelCase				)

           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(_lowerCAmelCase				)

             lowercase    :Dict      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase				)
             self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)

             lowercase    :Optional[int]      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					use_fast=_lowerCAmelCase				)
             self.assertIsInstance(_lowerCAmelCase       ,					_lowerCAmelCase				)

         finally:
           if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         # If remote code is not set, we will time out when asking whether to load the model.
         with self.assertRaises(_lowerCAmelCase				):
           lowercase    :List[str]      	=						AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"				)
         # If remote code is disabled, we can't load this config.
         with self.assertRaises(_lowerCAmelCase				):
           lowercase    :str      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase				)

         lowercase    :Tuple      	=						AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase				)
         self.assertTrue(tokenizer.special_attribute_present				)
         # Test tokenizer can be reloaded.
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCAmelCase				)
           lowercase    :Dict      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					trust_remote_code=_lowerCAmelCase				)
         self.assertTrue(reloaded_tokenizer.special_attribute_present				)

         if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizerFast"				)
           self.assertEqual(reloaded_tokenizer.__class__.__name__       ,					"NewTokenizerFast"				)

           # Test we can also load the slow version
           lowercase    :Optional[Any]      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase       ,					use_fast=_lowerCAmelCase				)
           self.assertTrue(tokenizer.special_attribute_present				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(_lowerCAmelCase				)
             lowercase    :Optional[int]      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					trust_remote_code=_lowerCAmelCase       ,					use_fast=_lowerCAmelCase				)
           self.assertEqual(reloaded_tokenizer.__class__.__name__       ,					"NewTokenizer"				)
           self.assertTrue(reloaded_tokenizer.special_attribute_present				)
         else:
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)
           self.assertEqual(reloaded_tokenizer.__class__.__name__       ,					"NewTokenizer"				)






       @require_tokenizers
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):




         class    __lowerCAmelCase       (							lowerCAmelCase):
                _a 					=						False







         class    __lowerCAmelCase       (							lowerCAmelCase):
                _a 					=						NewTokenizer
                _a 					=						False

         try:
           AutoConfig.register("custom"       ,					_lowerCAmelCase				)
           AutoTokenizer.register(_lowerCAmelCase       ,					slow_tokenizer_class=_lowerCAmelCase				)
           AutoTokenizer.register(_lowerCAmelCase       ,					fast_tokenizer_class=_lowerCAmelCase				)
           # If remote code is not set, the default is to use local
           lowercase    :Tuple      	=						AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizerFast"				)
           self.assertFalse(tokenizer.special_attribute_present				)
           lowercase    :int      	=						AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,					use_fast=_lowerCAmelCase				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)
           self.assertFalse(tokenizer.special_attribute_present				)

           # If remote code is disabled, we load the local one.
           lowercase    :int      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizerFast"				)
           self.assertFalse(tokenizer.special_attribute_present				)
           lowercase    :Any      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase       ,					use_fast=_lowerCAmelCase				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)
           self.assertFalse(tokenizer.special_attribute_present				)

           # If remote is enabled, we load from the Hub
           lowercase    :List[str]      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizerFast"				)
           self.assertTrue(tokenizer.special_attribute_present				)
           lowercase    :str      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"       ,					trust_remote_code=_lowerCAmelCase       ,					use_fast=_lowerCAmelCase				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)
           self.assertTrue(tokenizer.special_attribute_present				)

         finally:
           if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         lowercase    :Tuple      	=						AutoTokenizer.from_pretrained(
             "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,					trust_remote_code=_lowerCAmelCase				)
         self.assertTrue(tokenizer.special_attribute_present				)
         if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizerFast"				)

           # Test we can also load the slow version
           lowercase    :List[str]      	=						AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,					trust_remote_code=_lowerCAmelCase       ,					use_fast=_lowerCAmelCase				)
           self.assertTrue(tokenizer.special_attribute_present				)
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)
         else:
           self.assertEqual(tokenizer.__class__.__name__       ,					"NewTokenizer"				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         with self.assertRaisesRegex(
             _lowerCAmelCase       ,					"bert-base is not a local folder and is not a valid model identifier"				):
           lowercase    :Optional[int]      	=						AutoTokenizer.from_pretrained("bert-base"				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         with self.assertRaisesRegex(
             _lowerCAmelCase       ,					r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"				):
           lowercase    :Tuple      	=						AutoTokenizer.from_pretrained(_lowerCAmelCase       ,					revision="aaaaaa"				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         # Make sure we have cached the tokenizer.
         lowercase    :List[str]      	=						AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"				)
         with RequestCounter() as counter:
           lowercase    :List[str]      	=						AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"				)
           self.assertEqual(counter.get_request_count       ,					0				)
           self.assertEqual(counter.head_request_count       ,					1				)
           self.assertEqual(counter.other_request_count       ,					0				)









_UpperCAmelCase	:						Union[str, Any]			=     "0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
     from .utils import rich



