






'''simple docstring'''



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      def 				A__      ( self			:		Optional[Any]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   Tuple				  =mock.Mock()
       lowercase					:   Optional[Any]				  =500
       lowercase					:   Optional[Any]				  ={}
       lowercase					:   List[str]				  =HTTPError
       lowercase					:   List[str]				  ={}

       # Download this model to make sure it's in the cache.
       lowercase					:   Optional[Any]				  =BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('''requests.Session.request'''     ,      return_value=UpperCAmelCase		) as mock_head:
        lowercase					:   Optional[int]				  =BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
        # This check we did call the fake head request
        mock_head.assert_called()






      @require_tokenizers
      def 				A__      ( self			:		Any		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   int				  =mock.Mock()
       lowercase					:   Dict				  =500
       lowercase					:   List[Any]				  ={}
       lowercase					:   Union[str, Any]				  =HTTPError
       lowercase					:   str				  ={}

       # Download this model to make sure it's in the cache.
       lowercase					:   Optional[Any]				  =GPTaTokenizerFast.from_pretrained('''gpt2'''		)

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('''requests.Session.request'''     ,      return_value=UpperCAmelCase		) as mock_head:
        lowercase					:   Any				  =GPTaTokenizerFast.from_pretrained('''gpt2'''		)
        # This check we did call the fake head request
        mock_head.assert_called()






      def 				A__      ( self			:		Dict		)  ->  Any:


       '''simple docstring'''

       try:
        lowercase					:   int				  =tempfile.mktemp()
        with open(UpperCAmelCase     ,      '''wb'''		) as f:
         http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''     ,      UpperCAmelCase		)

        lowercase					:   Optional[int]				  =AlbertTokenizer.from_pretrained(UpperCAmelCase		)
       finally:
        os.remove(UpperCAmelCase		)

       # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
       # the current folder and have the right name.
       if os.path.isfile('''tokenizer.json'''		):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
       try:
        with open('''tokenizer.json'''     ,      '''wb'''		) as f:
         http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''     ,      UpperCAmelCase		)
        lowercase					:   Tuple				  =AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''		)
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size     ,      1000		)
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

       finally:
        os.remove('''tokenizer.json'''		)






      def 				A__      ( self			:		Optional[Any]		)  ->  Any:


       '''simple docstring'''

       lowercase					:   Dict				  =AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''		)





@is_staging_test
class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      UpperCamelCase_								=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






      @classmethod
      def 				A__      ( cls			:		Any		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   Any				  =TOKEN
       HfFolder.save_token(UpperCAmelCase		)






      @classmethod
      def 				A__      ( cls			:		Optional[Any]		)  ->  Union[str, Any]:


       '''simple docstring'''

       try:
        delete_repo(token=cls._token     ,      repo_id='''test-tokenizer'''		)
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token     ,      repo_id='''valid_org/test-tokenizer-org'''		)
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token     ,      repo_id='''test-dynamic-tokenizer'''		)
       except HTTPError:
        pass






      def 				A__      ( self			:		Union[str, Any]		)  ->  Tuple:


       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase					:   Union[str, Any]				  =os.path.join(UpperCAmelCase     ,      '''vocab.txt'''		)
        with open(UpperCAmelCase     ,      '''w'''     ,      encoding='''utf-8'''		) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
        lowercase					:   Optional[Any]				  =BertTokenizer(UpperCAmelCase		)

       tokenizer.push_to_hub('''test-tokenizer'''     ,      use_auth_token=self._token		)
       lowercase					:   Any				  =BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'		)
       self.assertDictEqual(new_tokenizer.vocab     ,      tokenizer.vocab		)

       # Reset repo
       delete_repo(token=self._token     ,      repo_id='''test-tokenizer'''		)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(UpperCAmelCase     ,      repo_id='''test-tokenizer'''     ,      push_to_hub=UpperCAmelCase     ,      use_auth_token=self._token		)

       lowercase					:   Any				  =BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'		)
       self.assertDictEqual(new_tokenizer.vocab     ,      tokenizer.vocab		)






      def 				A__      ( self			:		List[str]		)  ->  str:


       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase					:   str				  =os.path.join(UpperCAmelCase     ,      '''vocab.txt'''		)
        with open(UpperCAmelCase     ,      '''w'''     ,      encoding='''utf-8'''		) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
        lowercase					:   str				  =BertTokenizer(UpperCAmelCase		)

       tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''     ,      use_auth_token=self._token		)
       lowercase					:   List[Any]				  =BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
       self.assertDictEqual(new_tokenizer.vocab     ,      tokenizer.vocab		)

       # Reset repo
       delete_repo(token=self._token     ,      repo_id='''valid_org/test-tokenizer-org'''		)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            UpperCAmelCase     ,      repo_id='''valid_org/test-tokenizer-org'''     ,      push_to_hub=UpperCAmelCase     ,      use_auth_token=self._token		)

       lowercase					:   Optional[int]				  =BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
       self.assertDictEqual(new_tokenizer.vocab     ,      tokenizer.vocab		)






      @require_tokenizers
      def 				A__      ( self			:		Optional[Any]		)  ->  int:


       '''simple docstring'''

       CustomTokenizer.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase					:   List[Any]				  =os.path.join(UpperCAmelCase     ,      '''vocab.txt'''		)
        with open(UpperCAmelCase     ,      '''w'''     ,      encoding='''utf-8'''		) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
        lowercase					:   List[str]				  =CustomTokenizer(UpperCAmelCase		)

       # No fast custom tokenizer
       tokenizer.push_to_hub('''test-dynamic-tokenizer'''     ,      use_auth_token=self._token		)

       lowercase					:   Any				  =AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'     ,      trust_remote_code=UpperCAmelCase		)
       # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__     ,      '''CustomTokenizer'''		)

       # Fast and slow custom tokenizer
       CustomTokenizerFast.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
        lowercase					:   int				  =os.path.join(UpperCAmelCase     ,      '''vocab.txt'''		)
        with open(UpperCAmelCase     ,      '''w'''     ,      encoding='''utf-8'''		) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)

        lowercase					:   Union[str, Any]				  =BertTokenizerFast.from_pretrained(UpperCAmelCase		)
        bert_tokenizer.save_pretrained(UpperCAmelCase		)
        lowercase					:   Optional[Any]				  =CustomTokenizerFast.from_pretrained(UpperCAmelCase		)

       tokenizer.push_to_hub('''test-dynamic-tokenizer'''     ,      use_auth_token=self._token		)

       lowercase					:   List[Any]				  =AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'     ,      trust_remote_code=UpperCAmelCase		)
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__     ,      '''CustomTokenizerFast'''		)
       lowercase					:   Dict				  =AutoTokenizer.from_pretrained(
           f'{USER}/test-dynamic-tokenizer'     ,      use_fast=UpperCAmelCase     ,      trust_remote_code=UpperCAmelCase		)
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__     ,      '''CustomTokenizer'''		)





class UpperCAmelCase_					(    unittest.TestCase				):




      """simple docstring"""




      def 				A__      ( self			:		Optional[int]		)  ->  Any:


       '''simple docstring'''

       lowercase					:   List[str]				  =Trie()
       trie.add('''Hello 友達'''		)
       self.assertEqual(trie.data     ,      {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)
       trie.add('''Hello'''		)
       trie.data
       self.assertEqual(trie.data     ,      {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)






      def 				A__      ( self			:		Tuple		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   int				  =Trie()
       self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)     ,      ['''[CLS] This is a extra_id_100''']		)
       trie.add('''[CLS]'''		)
       trie.add('''extra_id_1'''		)
       trie.add('''extra_id_100'''		)
       self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)     ,      ['''[CLS]''', ''' This is a ''', '''extra_id_100''']		)






      def 				A__      ( self			:		Optional[int]		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   str				  =Trie()
       trie.add('''A'''		)
       self.assertEqual(trie.split('''ABC'''		)     ,      ['''A''', '''BC''']		)
       self.assertEqual(trie.split('''BCA'''		)     ,      ['''BC''', '''A''']		)






      def 				A__      ( self			:		Optional[int]		)  ->  Union[str, Any]:


       '''simple docstring'''

       lowercase					:   Tuple				  =Trie()
       trie.add('''TOKEN]'''		)
       trie.add('''[SPECIAL_TOKEN]'''		)
       self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)     ,      ['''This is something ''', '''[SPECIAL_TOKEN]''']		)






      def 				A__      ( self			:		Dict		)  ->  Tuple:


       '''simple docstring'''

       lowercase					:   Union[str, Any]				  =Trie()
       trie.add('''A'''		)
       trie.add('''P'''		)
       trie.add('''[SPECIAL_TOKEN]'''		)
       self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)     ,      ['''This is something ''', '''[SPECIAL_TOKEN]''']		)






      def 				A__      ( self			:		Tuple		)  ->  Optional[Any]:


       '''simple docstring'''

       lowercase					:   List[Any]				  =Trie()
       trie.add('''AB'''		)
       trie.add('''B'''		)
       trie.add('''C'''		)
       self.assertEqual(trie.split('''ABC'''		)     ,      ['''AB''', '''C''']		)






      def 				A__      ( self			:		List[str]		)  ->  List[str]:


       '''simple docstring'''

       lowercase					:   List[str]				  =Trie()
       trie.add('''ABC'''		)
       trie.add('''B'''		)
       trie.add('''CD'''		)
       self.assertEqual(trie.split('''ABCD'''		)     ,      ['''ABC''', '''D''']		)






      def 				A__      ( self			:		int		)  ->  Optional[int]:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =Trie()
       lowercase					:   Optional[int]				  =trie.cut_text('''ABC'''     ,      [0, 0, 2, 1, 2, 3]		)
       self.assertEqual(UpperCAmelCase     ,      ['''AB''', '''C''']		)











'''simple docstring'''



import importlib.metadata
from typing import Union

from packaging.version import Version, parse

from .constants import STR_OPERATION_TO_FUNC


SCREAMING_SNAKE_CASE						=	parse(importlib.metadata.version('torch'))


def 							lowercase_				(    __A :       Union[str, Version]					,      __A :       str					,      __A :       str			)       ->      Union[str, Any]:



 """simple docstring"""




 if operation not in STR_OPERATION_TO_FUNC.keys():
  raise ValueError(F'`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys()			)}, received {operation}'			)
 lowercase					:   Any				  =STR_OPERATION_TO_FUNC[operation]
 if isinstance(__A					,      __A			):
  lowercase					:   List[Any]				  =parse(importlib.metadata.version(__A			)			)
 return operation(__A					,      parse(__A			)			)





def 							lowercase_				(    __A :       str					,      __A :       str			)       ->      Tuple:



 """simple docstring"""




 return compare_versions(__A					,      __A					,      __A			)



