



'''simple docstring'''


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast







class 	UpperCAmelCase	(		unittest.TestCase):


 '''simple docstring'''




 def       lowercase_  (   self) ->      Union[str, Any]:



      """simple docstring"""







      a_       		=0




 @slow
 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
           a_       		=AutoTokenizer.from_pretrained(lowercase_)
           self.assertIsNotNone(lowercase_)
           self.assertIsInstance(lowercase_							,  (BertTokenizer, BertTokenizerFast))
           self.assertGreater(len(lowercase_)							,  0)

      for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
           a_       		=AutoTokenizer.from_pretrained(lowercase_)
           self.assertIsNotNone(lowercase_)
           self.assertIsInstance(lowercase_							,  (GPTaTokenizer, GPTaTokenizerFast))
           self.assertGreater(len(lowercase_)							,  0)




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained(lowercase_)
      self.assertIsInstance(lowercase_							,  (BertTokenizer, BertTokenizerFast))
      self.assertEqual(tokenizer.vocab_size							,  1_2)




 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained(lowercase_)
      self.assertIsInstance(lowercase_							,  (RobertaTokenizer, RobertaTokenizerFast))
      self.assertEqual(tokenizer.vocab_size							,  2_0)




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      a_       		=AutoConfig.from_pretrained(lowercase_)
      self.assertIsInstance(lowercase_							,  lowercase_)
      # Check that tokenizer_type ≠ model_type
      a_       		=AutoTokenizer.from_pretrained(lowercase_							,  config=lowercase_)
      self.assertIsInstance(lowercase_							,  (BertTokenizer, BertTokenizerFast))
      self.assertEqual(tokenizer.vocab_size							,  1_2)




 def       lowercase_  (   self) ->      str:



      """simple docstring"""







      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.txt"							,  os.path.join(lowercase_							,  "vocab.txt"))

           a_       		=AutoTokenizer.from_pretrained(lowercase_							,  tokenizer_type="bert"							,  use_fast=lowercase_)
           self.assertIsInstance(lowercase_							,  lowercase_)

      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.json"							,  os.path.join(lowercase_							,  "vocab.json"))
           shutil.copy("./tests/fixtures/merges.txt"							,  os.path.join(lowercase_							,  "merges.txt"))

           a_       		=AutoTokenizer.from_pretrained(lowercase_							,  tokenizer_type="gpt2"							,  use_fast=lowercase_)
           self.assertIsInstance(lowercase_							,  lowercase_)




 @require_tokenizers
 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.txt"							,  os.path.join(lowercase_							,  "vocab.txt"))

           a_       		=AutoTokenizer.from_pretrained(lowercase_							,  tokenizer_type="bert")
           self.assertIsInstance(lowercase_							,  lowercase_)

      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.json"							,  os.path.join(lowercase_							,  "vocab.json"))
           shutil.copy("./tests/fixtures/merges.txt"							,  os.path.join(lowercase_							,  "merges.txt"))

           a_       		=AutoTokenizer.from_pretrained(lowercase_							,  tokenizer_type="gpt2")
           self.assertIsInstance(lowercase_							,  lowercase_)




 def       lowercase_  (   self) ->      Union[str, Any]:



      """simple docstring"""







      with pytest.raises(lowercase_):
           AutoTokenizer.from_pretrained("./"							,  tokenizer_type="xxx")




 @require_tokenizers
 def       lowercase_  (   self) ->      str:



      """simple docstring"""







      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           a_       		=tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased")
           self.assertIsInstance(lowercase_							,  (BertTokenizer, BertTokenizerFast))

           if isinstance(lowercase_							,  lowercase_):
                self.assertEqual(tokenizer.basic_tokenizer.do_lower_case							,  lowercase_)
           else:
                self.assertEqual(tokenizer.do_lower_case							,  lowercase_)

           self.assertEqual(tokenizer.model_max_length							,  5_1_2)




 @require_tokenizers
 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           with self.assertRaisesRegex(
               lowercase_							,  "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"							,  ):
                a_       		=tokenizer_class.from_pretrained("julien-c/herlolip-not-exists")




 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      a_       		=TOKENIZER_MAPPING.values()
      a_       		=[]

      for slow_tok, fast_tok in tokenizers:
           if slow_tok is not None:
                tokenizer_names.append(slow_tok.__name__)

           if fast_tok is not None:
                tokenizer_names.append(fast_tok.__name__)

      for tokenizer_name in tokenizer_names:
           # must find the right class
           tokenizer_class_from_name(lowercase_)




 @require_tokenizers
 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"							,  use_fast=lowercase_)							,  lowercase_)
      self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased")							,  lowercase_)




 @require_tokenizers
 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained("distilbert-base-uncased"							,  do_lower_case=lowercase_)
      a_       		="""Hello, world. How are you?"""
      a_       		=tokenizer.tokenize(lowercase_)
      self.assertEqual("[UNK]"							,  tokens[0])

      a_       		=AutoTokenizer.from_pretrained("microsoft/mpnet-base"							,  do_lower_case=lowercase_)
      a_       		=tokenizer.tokenize(lowercase_)
      self.assertEqual("[UNK]"							,  tokens[0])




 @require_tokenizers
 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config")
      self.assertEqual(type(lowercase_)							,  lowercase_)
      self.assertEqual(tokenizer.model_max_length							,  5_1_2)
      self.assertEqual(tokenizer.vocab_size							,  3_0_0_0_0)
      self.assertEqual(tokenizer.unk_token							,  "[UNK]")
      self.assertEqual(tokenizer.padding_side							,  "right")
      self.assertEqual(tokenizer.truncation_side							,  "right")




 def       lowercase_  (   self) ->      Optional[int]:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained(lowercase_)
      self.assertIsInstance(lowercase_							,  (BertTokenizer, BertTokenizerFast))
      with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowercase_)
           a_       		=AutoTokenizer.from_pretrained(lowercase_)

      self.assertIsInstance(lowercase_							,  tokenizer.__class__)
      self.assertEqual(tokenizera.vocab_size							,  1_2)




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained("ctrl")
      # There is no fast CTRL so this always gives us a slow tokenizer.
      self.assertIsInstance(lowercase_							,  lowercase_)




 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      a_       		=get_tokenizer_config("bert-base-cased")
      a_       		=config.pop("_commit_hash"							,  lowercase_)
      # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
      self.assertEqual(lowercase_							,  {"do_lower_case": False})

      # This model does not have a tokenizer_config so we get back an empty dict.
      a_       		=get_tokenizer_config(lowercase_)
      self.assertDictEqual(lowercase_							,  {})

      # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
      a_       		=AutoTokenizer.from_pretrained(lowercase_)
      with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowercase_)
           a_       		=get_tokenizer_config(lowercase_)

      # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
      self.assertEqual(config["tokenizer_class"]							,  "BertTokenizer")




 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      try:
           AutoConfig.register("custom"							,  lowercase_)

           AutoTokenizer.register(lowercase_							,  slow_tokenizer_class=lowercase_)
           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(lowercase_):
                AutoTokenizer.register(lowercase_							,  slow_tokenizer_class=lowercase_)

           a_       		=CustomTokenizer.from_pretrained(lowercase_)
           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer.save_pretrained(lowercase_)

                a_       		=AutoTokenizer.from_pretrained(lowercase_)
                self.assertIsInstance(lowercase_							,  lowercase_)

      finally:
           if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
                del TOKENIZER_MAPPING._extra_content[CustomConfig]




 @require_tokenizers
 def       lowercase_  (   self) ->      str:



      """simple docstring"""







      try:
           AutoConfig.register("custom"							,  lowercase_)

           # Can register in two steps
           AutoTokenizer.register(lowercase_							,  slow_tokenizer_class=lowercase_)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]							,  (CustomTokenizer, None))
           AutoTokenizer.register(lowercase_							,  fast_tokenizer_class=lowercase_)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]							,  (CustomTokenizer, CustomTokenizerFast))

           del TOKENIZER_MAPPING._extra_content[CustomConfig]
           # Can register in one step
           AutoTokenizer.register(
               lowercase_							,  slow_tokenizer_class=lowercase_							,  fast_tokenizer_class=lowercase_)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]							,  (CustomTokenizer, CustomTokenizerFast))

           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(lowercase_):
                AutoTokenizer.register(lowercase_							,  fast_tokenizer_class=lowercase_)

           # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
           # and that model does not have a tokenizer.json
           with tempfile.TemporaryDirectory() as tmp_dir:
                a_       		=BertTokenizerFast.from_pretrained(lowercase_)
                bert_tokenizer.save_pretrained(lowercase_)
                a_       		=CustomTokenizerFast.from_pretrained(lowercase_)

           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer.save_pretrained(lowercase_)

                a_       		=AutoTokenizer.from_pretrained(lowercase_)
                self.assertIsInstance(lowercase_							,  lowercase_)

                a_       		=AutoTokenizer.from_pretrained(lowercase_							,  use_fast=lowercase_)
                self.assertIsInstance(lowercase_							,  lowercase_)

      finally:
           if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
                del TOKENIZER_MAPPING._extra_content[CustomConfig]




 def       lowercase_  (   self) ->      Union[str, Any]:



      """simple docstring"""







      with self.assertRaises(lowercase_):
           a_       		=AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer")
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(lowercase_):
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_)

      a_       		=AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_)
      self.assertTrue(tokenizer.special_attribute_present)
      # Test tokenizer can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowercase_)
           a_       		=AutoTokenizer.from_pretrained(lowercase_							,  trust_remote_code=lowercase_)
      self.assertTrue(reloaded_tokenizer.special_attribute_present)

      if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizerFast")
           self.assertEqual(reloaded_tokenizer.__class__.__name__							,  "NewTokenizerFast")

           # Test we can also load the slow version
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_							,  use_fast=lowercase_)
           self.assertTrue(tokenizer.special_attribute_present)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer.save_pretrained(lowercase_)
                a_       		=AutoTokenizer.from_pretrained(lowercase_							,  trust_remote_code=lowercase_							,  use_fast=lowercase_)
           self.assertEqual(reloaded_tokenizer.__class__.__name__							,  "NewTokenizer")
           self.assertTrue(reloaded_tokenizer.special_attribute_present)
      else:
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")
           self.assertEqual(reloaded_tokenizer.__class__.__name__							,  "NewTokenizer")




 @require_tokenizers
 def       lowercase_  (   self) ->      str:



      """simple docstring"""







      class 	UpperCAmelCase	(		_UpperCAmelCase):


       '''simple docstring'''




       __magic_name__					: List[str]  =      False







      class 	UpperCAmelCase	(		_UpperCAmelCase):


       '''simple docstring'''




       __magic_name__					: Union[str, Any]  =      NewTokenizer
       __magic_name__					: Tuple  =      False

      try:
           AutoConfig.register("custom"							,  lowercase_)
           AutoTokenizer.register(lowercase_							,  slow_tokenizer_class=lowercase_)
           AutoTokenizer.register(lowercase_							,  fast_tokenizer_class=lowercase_)
           # If remote code is not set, the default is to use local
           a_       		=AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer")
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizerFast")
           self.assertFalse(tokenizer.special_attribute_present)
           a_       		=AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"							,  use_fast=lowercase_)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")
           self.assertFalse(tokenizer.special_attribute_present)

           # If remote code is disabled, we load the local one.
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizerFast")
           self.assertFalse(tokenizer.special_attribute_present)
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_							,  use_fast=lowercase_)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")
           self.assertFalse(tokenizer.special_attribute_present)

           # If remote is enabled, we load from the Hub
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizerFast")
           self.assertTrue(tokenizer.special_attribute_present)
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"							,  trust_remote_code=lowercase_							,  use_fast=lowercase_)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")
           self.assertTrue(tokenizer.special_attribute_present)

      finally:
           if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
                del TOKENIZER_MAPPING._extra_content[CustomConfig]




 def       lowercase_  (   self) ->      int:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained(
          "hf-internal-testing/test_dynamic_tokenizer_legacy"							,  trust_remote_code=lowercase_)
      self.assertTrue(tokenizer.special_attribute_present)
      if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizerFast")

           # Test we can also load the slow version
           a_       		=AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer_legacy"							,  trust_remote_code=lowercase_							,  use_fast=lowercase_)
           self.assertTrue(tokenizer.special_attribute_present)
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")
      else:
           self.assertEqual(tokenizer.__class__.__name__							,  "NewTokenizer")




 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      with self.assertRaisesRegex(
          lowercase_							,  "bert-base is not a local folder and is not a valid model identifier"):
           a_       		=AutoTokenizer.from_pretrained("bert-base")




 def       lowercase_  (   self) ->      Union[str, Any]:



      """simple docstring"""







      with self.assertRaisesRegex(
          lowercase_							,  r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"):
           a_       		=AutoTokenizer.from_pretrained(lowercase_							,  revision="aaaaaa")




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      a_       		=AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert")
      with RequestCounter() as counter:
           a_       		=AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert")
           self.assertEqual(counter.get_request_count							,  0)
           self.assertEqual(counter.head_request_count							,  1)
           self.assertEqual(counter.other_request_count							,  0)







'''simple docstring'''


from collections.abc import Generator






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_		,						a_       		=0, 1
     while True:
          a_		,						a_       		=b, a + b
          yield b






def  UpperCAmelCase_       (		lowercase__ = 1_0_0_0       ):


     '''simple docstring'''







     a_       		=1
     a_       		=fibonacci_generator()
     while len(str(next(lowercase__       )       )       ) < n:
          answer += 1
     return answer + 1


if __name__ == "__main__":
  print(solution(int(str(input()).strip())))


