
"""simple docstring"""





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast



class __lowerCAmelCase				(					unittest.TestCase      ):




			"""simple docstring"""







			def      lowerCamelCase__     (				self						:					Tuple	)      -> int:







								"""simple docstring"""
								# A mock response for an HTTP head request to emulate server down
								A_				 =						mock.Mock()
								A_				 =						500
								A_				 =						{}
								A_				 =						HTTPError
								A_				 =						{}

								# Download this model to make sure it's in the cache.
								A_				 =						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"	)

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch("requests.Session.request"       ,   return_value=_snake_case	) as mock_head:
													A_				 =						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"	)
													# This check we did call the fake head request
													mock_head.assert_called()




			@require_tokenizers
			def      lowerCamelCase__     (				self						:					Any	)      -> Dict:







								"""simple docstring"""
								# A mock response for an HTTP head request to emulate server down
								A_				 =						mock.Mock()
								A_				 =						500
								A_				 =						{}
								A_				 =						HTTPError
								A_				 =						{}

								# Download this model to make sure it's in the cache.
								A_				 =						GPTaTokenizerFast.from_pretrained("gpt2"	)

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch("requests.Session.request"       ,   return_value=_snake_case	) as mock_head:
													A_				 =						GPTaTokenizerFast.from_pretrained("gpt2"	)
													# This check we did call the fake head request
													mock_head.assert_called()




			def      lowerCamelCase__     (				self						:					Any	)      -> Optional[int]:







								"""simple docstring"""
								# This test is for deprecated behavior and can be removed in v5
								try:
													A_				 =						tempfile.mktemp()
													with open(_snake_case       ,   "wb"	) as f:
																		http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       ,   _snake_case	)

													A_				 =						AlbertTokenizer.from_pretrained(_snake_case	)
								finally:
													os.remove(_snake_case	)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile("tokenizer.json"	):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
								try:
													with open("tokenizer.json"       ,   "wb"	) as f:
																		http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"       ,   _snake_case	)
													A_				 =						AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"	)
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size       ,   1_000	)
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
													os.remove("tokenizer.json"	)






			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Any:







								"""simple docstring"""
								# This test is for deprecated behavior and can be removed in v5
								A_				 =						AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"	)





@is_staging_test
class __lowerCAmelCase				(					unittest.TestCase      ):




			"""simple docstring"""







			snake_case	      =				["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




			@classmethod
			def      lowerCamelCase__     (				cls						:					str	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						TOKEN
								HfFolder.save_token(_snake_case	)




			@classmethod
			def      lowerCamelCase__     (				cls						:					Tuple	)      -> Union[str, Any]:







								"""simple docstring"""
								try:
													delete_repo(token=cls._token       ,   repo_id="test-tokenizer"	)
								except HTTPError:
													pass

								try:
													delete_repo(token=cls._token       ,   repo_id="valid_org/test-tokenizer-org"	)
								except HTTPError:
													pass

								try:
													delete_repo(token=cls._token       ,   repo_id="test-dynamic-tokenizer"	)
								except HTTPError:
													pass




			def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> List[str]:







								"""simple docstring"""
								with tempfile.TemporaryDirectory() as tmp_dir:
													A_				 =						os.path.join(_snake_case       ,   "vocab.txt"	)
													with open(_snake_case       ,   "w"       ,   encoding="utf-8"	) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]	)	)
													A_				 =						BertTokenizer(_snake_case	)

								tokenizer.push_to_hub("test-tokenizer"       ,   use_auth_token=self._token	)
								A_				 =						BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'	)
								self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab	)

								# Reset repo
								delete_repo(token=self._token       ,   repo_id="test-tokenizer"	)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_snake_case       ,   repo_id="test-tokenizer"       ,   push_to_hub=_snake_case       ,   use_auth_token=self._token	)

								A_				 =						BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'	)
								self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab	)




			def      lowerCamelCase__     (				self						:					int	)      -> Any:







								"""simple docstring"""
								with tempfile.TemporaryDirectory() as tmp_dir:
													A_				 =						os.path.join(_snake_case       ,   "vocab.txt"	)
													with open(_snake_case       ,   "w"       ,   encoding="utf-8"	) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]	)	)
													A_				 =						BertTokenizer(_snake_case	)

								tokenizer.push_to_hub("valid_org/test-tokenizer-org"       ,   use_auth_token=self._token	)
								A_				 =						BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"	)
								self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab	)

								# Reset repo
								delete_repo(token=self._token       ,   repo_id="valid_org/test-tokenizer-org"	)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    _snake_case       ,   repo_id="valid_org/test-tokenizer-org"       ,   push_to_hub=_snake_case       ,   use_auth_token=self._token	)

								A_				 =						BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"	)
								self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab	)






			@require_tokenizers
			def      lowerCamelCase__     (				self						:					List[str]	)      -> Any:







								"""simple docstring"""
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
													A_				 =						os.path.join(_snake_case       ,   "vocab.txt"	)
													with open(_snake_case       ,   "w"       ,   encoding="utf-8"	) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]	)	)
													A_				 =						CustomTokenizer(_snake_case	)

								# No fast custom tokenizer
								tokenizer.push_to_hub("test-dynamic-tokenizer"       ,   use_auth_token=self._token	)

								A_				 =						AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'       ,   trust_remote_code=_snake_case	)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,   "CustomTokenizer"	)

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
													A_				 =						os.path.join(_snake_case       ,   "vocab.txt"	)
													with open(_snake_case       ,   "w"       ,   encoding="utf-8"	) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]	)	)

													A_				 =						BertTokenizerFast.from_pretrained(_snake_case	)
													bert_tokenizer.save_pretrained(_snake_case	)
													A_				 =						CustomTokenizerFast.from_pretrained(_snake_case	)

								tokenizer.push_to_hub("test-dynamic-tokenizer"       ,   use_auth_token=self._token	)

								A_				 =						AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'       ,   trust_remote_code=_snake_case	)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,   "CustomTokenizerFast"	)
								A_				 =						AutoTokenizer.from_pretrained(
								    F'{USER}/test-dynamic-tokenizer'       ,   use_fast=_snake_case       ,   trust_remote_code=_snake_case	)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__       ,   "CustomTokenizer"	)





class __lowerCAmelCase				(					unittest.TestCase      ):




			"""simple docstring"""







			def      lowerCamelCase__     (				self						:					List[str]	)      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						Trie()
								trie.add("Hello 友達"	)
								self.assertEqual(trie.data       ,   {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}	)
								trie.add("Hello"	)
								trie.data
								self.assertEqual(trie.data       ,   {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}	)




			def      lowerCamelCase__     (				self						:					Any	)      -> Any:







								"""simple docstring"""
								A_				 =						Trie()
								self.assertEqual(trie.split("[CLS] This is a extra_id_100"	)       ,   ["[CLS] This is a extra_id_100"]	)
								trie.add("[CLS]"	)
								trie.add("extra_id_1"	)
								trie.add("extra_id_100"	)
								self.assertEqual(trie.split("[CLS] This is a extra_id_100"	)       ,   ["[CLS]", " This is a ", "extra_id_100"]	)




			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Any:







								"""simple docstring"""
								A_				 =						Trie()
								trie.add("A"	)
								self.assertEqual(trie.split("ABC"	)       ,   ["A", "BC"]	)
								self.assertEqual(trie.split("BCA"	)       ,   ["BC", "A"]	)




			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						Trie()
								trie.add("TOKEN]"	)
								trie.add("[SPECIAL_TOKEN]"	)
								self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"	)       ,   ["This is something ", "[SPECIAL_TOKEN]"]	)




			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						Trie()
								trie.add("A"	)
								trie.add("P"	)
								trie.add("[SPECIAL_TOKEN]"	)
								self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"	)       ,   ["This is something ", "[SPECIAL_TOKEN]"]	)




			def      lowerCamelCase__     (				self						:					List[str]	)      -> str:







								"""simple docstring"""
								A_				 =						Trie()
								trie.add("AB"	)
								trie.add("B"	)
								trie.add("C"	)
								self.assertEqual(trie.split("ABC"	)       ,   ["AB", "C"]	)




			def      lowerCamelCase__     (				self						:					str	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						Trie()
								trie.add("ABC"	)
								trie.add("B"	)
								trie.add("CD"	)
								self.assertEqual(trie.split("ABCD"	)       ,   ["ABC", "D"]	)






			def      lowerCamelCase__     (				self						:					Dict	)      -> Dict:







								"""simple docstring"""
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								A_				 =						Trie()
								A_				 =						trie.cut_text("ABC"       ,   [0, 0, 2, 1, 2, 3]	)
								self.assertEqual(_snake_case       ,   ["AB", "C"]	)






"""simple docstring"""





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


UpperCamelCase_					:						int				= '''
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
'''



class __lowerCAmelCase				(					unittest.TestCase     ,				_lowercase      ):




			"""simple docstring"""







			def      lowerCamelCase__     (				self						:					str	)      -> Dict:







								"""simple docstring"""
								A_				 =						load_tool("text-question-answering"	)
								self.tool.setup()
								A_				 =						load_tool("text-question-answering"       ,   remote=_snake_case	)




			def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> Any:







								"""simple docstring"""
								A_				 =						self.tool(_snake_case       ,   "What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)




			def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> int:







								"""simple docstring"""
								A_				 =						self.remote_tool(_snake_case       ,   "What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)




			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						self.tool(text=_snake_case       ,   question="What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)






			def      lowerCamelCase__     (				self						:					str	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						self.remote_tool(text=_snake_case       ,   question="What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)




