


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast






class      UpperCamelCase					(		unittest.TestCase     ):


   '''simple docstring'''







   def 		UpperCamelCase  (     self		):
        # A mock response for an HTTP head request to emulate server down
        lowercase_       :str     =       mock.Mock()
        lowercase_       :Tuple     =       500
        lowercase_       :Union[str, Any]     =       {}
        lowercase_       :List[str]     =       HTTPError
        lowercase_       :List[str]     =       {}

        # Download this model to make sure it's in the cache.
        lowercase_       :Union[str, Any]     =       BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''   ,							return_value=UpperCamelCase_		) as mock_head:
             lowercase_       :Tuple     =       BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
             # This check we did call the fake head request
             mock_head.assert_called()
   @require_tokenizers
   def 		UpperCamelCase  (     self		):
        # A mock response for an HTTP head request to emulate server down
        lowercase_       :List[str]     =       mock.Mock()
        lowercase_       :Union[str, Any]     =       500
        lowercase_       :Dict     =       {}
        lowercase_       :Optional[int]     =       HTTPError
        lowercase_       :str     =       {}

        # Download this model to make sure it's in the cache.
        lowercase_       :Tuple     =       GPTaTokenizerFast.from_pretrained('''gpt2'''		)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''   ,							return_value=UpperCamelCase_		) as mock_head:
             lowercase_       :Any     =       GPTaTokenizerFast.from_pretrained('''gpt2'''		)
             # This check we did call the fake head request
             mock_head.assert_called()
   def 		UpperCamelCase  (     self		):
        # This test is for deprecated behavior and can be removed in v5
        try:
             lowercase_       :Tuple     =       tempfile.mktemp()
             with open(UpperCamelCase_   ,							'''wb'''		) as f:
                  http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   ,							UpperCamelCase_		)

             lowercase_       :str     =       AlbertTokenizer.from_pretrained(UpperCamelCase_		)
        finally:
             os.remove(UpperCamelCase_		)

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('''tokenizer.json'''		):
             # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
             return
        try:
             with open('''tokenizer.json'''   ,							'''wb'''		) as f:
                  http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''   ,							UpperCamelCase_		)
             lowercase_       :Union[str, Any]     =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''		)
             # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
             self.assertEqual(tokenizer.vocab_size   ,							1000		)
             # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
             os.remove('''tokenizer.json'''		)




   def 		UpperCamelCase  (     self		):
        # This test is for deprecated behavior and can be removed in v5
        lowercase_       :Union[str, Any]     =       AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''		)






@is_staging_test
class      UpperCamelCase					(		unittest.TestCase     ):


   '''simple docstring'''







   lowercase					:	Optional[int]									=["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]
   @classmethod
   def 		UpperCamelCase  (     cls		):
        lowercase_       :List[Any]     =       TOKEN
        HfFolder.save_token(UpperCamelCase_		)
   @classmethod
   def 		UpperCamelCase  (     cls		):
        try:
             delete_repo(token=cls._token   ,							repo_id='''test-tokenizer'''		)
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token   ,							repo_id='''valid_org/test-tokenizer-org'''		)
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token   ,							repo_id='''test-dynamic-tokenizer'''		)
        except HTTPError:
             pass
   def 		UpperCamelCase  (     self		):
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_       :Any     =       os.path.join(UpperCamelCase_   ,							'''vocab.txt'''		)
             with open(UpperCamelCase_   ,							'''w'''   ,							encoding='''utf-8'''		) as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
             lowercase_       :int     =       BertTokenizer(UpperCamelCase_		)

        tokenizer.push_to_hub('''test-tokenizer'''   ,							use_auth_token=self._token		)
        lowercase_       :Tuple     =       BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"		)
        self.assertDictEqual(new_tokenizer.vocab   ,							tokenizer.vocab		)

        # Reset repo
        delete_repo(token=self._token   ,							repo_id='''test-tokenizer'''		)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(UpperCamelCase_   ,							repo_id='''test-tokenizer'''   ,							push_to_hub=UpperCamelCase_   ,							use_auth_token=self._token		)

        lowercase_       :Dict     =       BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"		)
        self.assertDictEqual(new_tokenizer.vocab   ,							tokenizer.vocab		)
   def 		UpperCamelCase  (     self		):
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_       :Optional[int]     =       os.path.join(UpperCamelCase_   ,							'''vocab.txt'''		)
             with open(UpperCamelCase_   ,							'''w'''   ,							encoding='''utf-8'''		) as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
             lowercase_       :Union[str, Any]     =       BertTokenizer(UpperCamelCase_		)

        tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''   ,							use_auth_token=self._token		)
        lowercase_       :Union[str, Any]     =       BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
        self.assertDictEqual(new_tokenizer.vocab   ,							tokenizer.vocab		)

        # Reset repo
        delete_repo(token=self._token   ,							repo_id='''valid_org/test-tokenizer-org'''		)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(
                 UpperCamelCase_   ,							repo_id='''valid_org/test-tokenizer-org'''   ,							push_to_hub=UpperCamelCase_   ,							use_auth_token=self._token		)

        lowercase_       :Any     =       BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
        self.assertDictEqual(new_tokenizer.vocab   ,							tokenizer.vocab		)




   @require_tokenizers
   def 		UpperCamelCase  (     self		):
        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_       :Any     =       os.path.join(UpperCamelCase_   ,							'''vocab.txt'''		)
             with open(UpperCamelCase_   ,							'''w'''   ,							encoding='''utf-8'''		) as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
             lowercase_       :Optional[Any]     =       CustomTokenizer(UpperCamelCase_		)

        # No fast custom tokenizer
        tokenizer.push_to_hub('''test-dynamic-tokenizer'''   ,							use_auth_token=self._token		)

        lowercase_       :int     =       AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"   ,							trust_remote_code=UpperCamelCase_		)
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__   ,							'''CustomTokenizer'''		)

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_       :Optional[int]     =       os.path.join(UpperCamelCase_   ,							'''vocab.txt'''		)
             with open(UpperCamelCase_   ,							'''w'''   ,							encoding='''utf-8'''		) as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)

             lowercase_       :Union[str, Any]     =       BertTokenizerFast.from_pretrained(UpperCamelCase_		)
             bert_tokenizer.save_pretrained(UpperCamelCase_		)
             lowercase_       :Optional[int]     =       CustomTokenizerFast.from_pretrained(UpperCamelCase_		)

        tokenizer.push_to_hub('''test-dynamic-tokenizer'''   ,							use_auth_token=self._token		)

        lowercase_       :Union[str, Any]     =       AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"   ,							trust_remote_code=UpperCamelCase_		)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__   ,							'''CustomTokenizerFast'''		)
        lowercase_       :Dict     =       AutoTokenizer.from_pretrained(
            f"{USER}/test-dynamic-tokenizer"   ,							use_fast=UpperCamelCase_   ,							trust_remote_code=UpperCamelCase_		)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__   ,							'''CustomTokenizer'''		)






class      UpperCamelCase					(		unittest.TestCase     ):


   '''simple docstring'''







   def 		UpperCamelCase  (     self		):
        lowercase_       :Optional[int]     =       Trie()
        trie.add('''Hello 友達'''		)
        self.assertEqual(trie.data   ,							{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)
        trie.add('''Hello'''		)
        trie.data
        self.assertEqual(trie.data   ,							{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :Optional[Any]     =       Trie()
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)   ,							['''[CLS] This is a extra_id_100''']		)
        trie.add('''[CLS]'''		)
        trie.add('''extra_id_1'''		)
        trie.add('''extra_id_100'''		)
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)   ,							['''[CLS]''', ''' This is a ''', '''extra_id_100''']		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :List[str]     =       Trie()
        trie.add('''A'''		)
        self.assertEqual(trie.split('''ABC'''		)   ,							['''A''', '''BC''']		)
        self.assertEqual(trie.split('''BCA'''		)   ,							['''BC''', '''A''']		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :List[Any]     =       Trie()
        trie.add('''TOKEN]'''		)
        trie.add('''[SPECIAL_TOKEN]'''		)
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)   ,							['''This is something ''', '''[SPECIAL_TOKEN]''']		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :str     =       Trie()
        trie.add('''A'''		)
        trie.add('''P'''		)
        trie.add('''[SPECIAL_TOKEN]'''		)
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)   ,							['''This is something ''', '''[SPECIAL_TOKEN]''']		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :Tuple     =       Trie()
        trie.add('''AB'''		)
        trie.add('''B'''		)
        trie.add('''C'''		)
        self.assertEqual(trie.split('''ABC'''		)   ,							['''AB''', '''C''']		)
   def 		UpperCamelCase  (     self		):
        lowercase_       :List[str]     =       Trie()
        trie.add('''ABC'''		)
        trie.add('''B'''		)
        trie.add('''CD'''		)
        self.assertEqual(trie.split('''ABCD'''		)   ,							['''ABC''', '''D''']		)




   def 		UpperCamelCase  (     self		):
        # Even if the offsets are wrong, we necessarily output correct string
        # parts.
        lowercase_       :List[Any]     =       Trie()
        lowercase_       :str     =       trie.cut_text('''ABC'''   ,							[0, 0, 2, 1, 2, 3]		)
        self.assertEqual(UpperCamelCase_   ,							['''AB''', '''C''']		)





from torch import nn




def     UpperCamelCase    (      _a		)					->							str:
     '''simple docstring'''





     if act_fn in ["swish", "silu"]:
          return nn.SiLU()
     elif act_fn == "mish":
          return nn.Mish()
     elif act_fn == "gelu":
          return nn.GELU()
     else:
          raise ValueError(f"Unsupported activation function: {act_fn}"		)

