



import pprint

import requests

a_						:     Dict	     =		'https://zenquotes.io/api'

def     __lowercase(	):


      """simple docstring"""



      return requests.get(API_ENDPOINT_URL + "/today" ).json()

def     __lowercase(	):


      """simple docstring"""



      return requests.get(API_ENDPOINT_URL + "/random" ).json()


if __name__ == "__main__":
     a_						:     str	     =		random_quotes()
     pprint.pprint(response)




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast





class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		mock.Mock()
           lowerCamelCase								=		5_00
           lowerCamelCase								=		{}
           lowerCamelCase								=		HTTPError
           lowerCamelCase								=		{}

           # Download this model to make sure it's in the cache.
           lowerCamelCase								=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"    )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("requests.Session.request"				,			return_value=__a    ) as mock_head:
                 lowerCamelCase								=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"    )
                 # This check we did call the fake head request
                 mock_head.assert_called()







     @require_tokenizers
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		mock.Mock()
           lowerCamelCase								=		5_00
           lowerCamelCase								=		{}
           lowerCamelCase								=		HTTPError
           lowerCamelCase								=		{}

           # Download this model to make sure it's in the cache.
           lowerCamelCase								=		GPTaTokenizerFast.from_pretrained("gpt2"    )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("requests.Session.request"				,			return_value=__a    ) as mock_head:
                 lowerCamelCase								=		GPTaTokenizerFast.from_pretrained("gpt2"    )
                 # This check we did call the fake head request
                 mock_head.assert_called()







     def 						_a      (self    ):






           '''simple docstring'''



           try:
                 lowerCamelCase								=		tempfile.mktemp()
                 with open(__a				,			"wb"    ) as f:
                       http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"				,			__a    )

                 lowerCamelCase								=		AlbertTokenizer.from_pretrained(__a    )
           finally:
                 os.remove(__a    )

           # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
           # the current folder and have the right name.
           if os.path.isfile("tokenizer.json"    ):
                 # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                 return
           try:
                 with open("tokenizer.json"				,			"wb"    ) as f:
                       http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"				,			__a    )
                 lowerCamelCase								=		AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"    )
                 # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                 self.assertEqual(tokenizer.vocab_size				,			10_00    )
                 # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

           finally:
                 os.remove("tokenizer.json"    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"    )





@is_staging_test
class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     _A       =			['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']







     @classmethod
     def 						_a      (cls    ):






           '''simple docstring'''



           lowerCamelCase								=		TOKEN
           HfFolder.save_token(__a    )







     @classmethod
     def 						_a      (cls    ):






           '''simple docstring'''



           try:
                 delete_repo(token=cls._token				,			repo_id="test-tokenizer"    )
           except HTTPError:
                 pass

           try:
                 delete_repo(token=cls._token				,			repo_id="valid_org/test-tokenizer-org"    )
           except HTTPError:
                 pass

           try:
                 delete_repo(token=cls._token				,			repo_id="test-dynamic-tokenizer"    )
           except HTTPError:
                 pass







     def 						_a      (self    ):






           '''simple docstring'''



           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		BertTokenizer(__a    )

           tokenizer.push_to_hub("test-tokenizer"				,			use_auth_token=self._token    )
           lowerCamelCase								=		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )

           # Reset repo
           delete_repo(token=self._token				,			repo_id="test-tokenizer"    )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__a				,			repo_id="test-tokenizer"				,			push_to_hub=__a				,			use_auth_token=self._token    )

           lowerCamelCase								=		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )







     def 						_a      (self    ):






           '''simple docstring'''



           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		BertTokenizer(__a    )

           tokenizer.push_to_hub("valid_org/test-tokenizer-org"				,			use_auth_token=self._token    )
           lowerCamelCase								=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )

           # Reset repo
           delete_repo(token=self._token				,			repo_id="valid_org/test-tokenizer-org"    )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(
                     __a				,			repo_id="valid_org/test-tokenizer-org"				,			push_to_hub=__a				,			use_auth_token=self._token    )

           lowerCamelCase								=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )







     @require_tokenizers
     def 						_a      (self    ):






           '''simple docstring'''



           CustomTokenizer.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		CustomTokenizer(__a    )

           # No fast custom tokenizer
           tokenizer.push_to_hub("test-dynamic-tokenizer"				,			use_auth_token=self._token    )

           lowerCamelCase								=		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizer"    )

           # Fast and slow custom tokenizer
           CustomTokenizerFast.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )

                 lowerCamelCase								=		BertTokenizerFast.from_pretrained(__a    )
                 bert_tokenizer.save_pretrained(__a    )
                 lowerCamelCase								=		CustomTokenizerFast.from_pretrained(__a    )

           tokenizer.push_to_hub("test-dynamic-tokenizer"				,			use_auth_token=self._token    )

           lowerCamelCase								=		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizerFast"    )
           lowerCamelCase								=		AutoTokenizer.from_pretrained(
               F"""{USER}/test-dynamic-tokenizer"""				,			use_fast=__a				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizer"    )





class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("Hello 友達"    )
           self.assertEqual(trie.data				,			{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}    )
           trie.add("Hello"    )
           trie.data
           self.assertEqual(trie.data				,			{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           self.assertEqual(trie.split("[CLS] This is a extra_id_100"    )				,			["[CLS] This is a extra_id_100"]    )
           trie.add("[CLS]"    )
           trie.add("extra_id_1"    )
           trie.add("extra_id_100"    )
           self.assertEqual(trie.split("[CLS] This is a extra_id_100"    )				,			["[CLS]", " This is a ", "extra_id_100"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("A"    )
           self.assertEqual(trie.split("ABC"    )				,			["A", "BC"]    )
           self.assertEqual(trie.split("BCA"    )				,			["BC", "A"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("TOKEN]"    )
           trie.add("[SPECIAL_TOKEN]"    )
           self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"    )				,			["This is something ", "[SPECIAL_TOKEN]"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("A"    )
           trie.add("P"    )
           trie.add("[SPECIAL_TOKEN]"    )
           self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"    )				,			["This is something ", "[SPECIAL_TOKEN]"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("AB"    )
           trie.add("B"    )
           trie.add("C"    )
           self.assertEqual(trie.split("ABC"    )				,			["AB", "C"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("ABC"    )
           trie.add("B"    )
           trie.add("CD"    )
           self.assertEqual(trie.split("ABCD"    )				,			["ABC", "D"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           lowerCamelCase								=		trie.cut_text("ABC"				,			[0, 0, 2, 1, 2, 3]    )
           self.assertEqual(__a				,			["AB", "C"]    )