


'''simple docstring'''



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast






class  UpperCamelCase__(     unittest.TestCase ):




				def   a__(	self    :				List[Any]   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			mock.Mock()
										UpperCAmelCase         =			500
										UpperCAmelCase         =			{}
										UpperCAmelCase         =			HTTPError
										UpperCAmelCase         =			{}

										# Download this model to make sure it's in the cache.
										UpperCAmelCase         =			BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('''requests.Session.request'''   ,					return_value=lowerCAmelCase   ) as mock_head:
																UpperCAmelCase         =			BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )
																# This check we did call the fake head request
																mock_head.assert_called()




				@require_tokenizers
				def   a__(	self    :				int   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			mock.Mock()
										UpperCAmelCase         =			500
										UpperCAmelCase         =			{}
										UpperCAmelCase         =			HTTPError
										UpperCAmelCase         =			{}

										# Download this model to make sure it's in the cache.
										UpperCAmelCase         =			GPTaTokenizerFast.from_pretrained('''gpt2'''   )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch('''requests.Session.request'''   ,					return_value=lowerCAmelCase   ) as mock_head:
																UpperCAmelCase         =			GPTaTokenizerFast.from_pretrained('''gpt2'''   )
																# This check we did call the fake head request
																mock_head.assert_called()




				def   a__(	self    :				str   )->  Optional[Any]:

										"""simple docstring"""




										try:
																UpperCAmelCase         =			tempfile.mktemp()
																with open(lowerCAmelCase   ,					'''wb'''   ) as f:
																						http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   ,					lowerCAmelCase   )

																UpperCAmelCase         =			AlbertTokenizer.from_pretrained(lowerCAmelCase   )
										finally:
																os.remove(lowerCAmelCase   )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile('''tokenizer.json'''   ):
																# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																return
										try:
																with open('''tokenizer.json'''   ,					'''wb'''   ) as f:
																						http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''   ,					lowerCAmelCase   )
																UpperCAmelCase         =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''   )
																# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																self.assertEqual(tokenizer.vocab_size   ,					1000   )
																# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
																os.remove('''tokenizer.json'''   )





				def   a__(	self    :				Optional[Any]   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   )







@is_staging_test
class  UpperCamelCase__(     unittest.TestCase ):
				__magic_name__       :   List[str]							= ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




				@classmethod
				def   a__(	cls    :				Optional[int]   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			TOKEN
										HfFolder.save_token(lowerCAmelCase   )




				@classmethod
				def   a__(	cls    :				Optional[Any]   )->  Optional[Any]:

										"""simple docstring"""




										try:
																delete_repo(token=cls._token   ,					repo_id='''test-tokenizer'''   )
										except HTTPError:
																pass

										try:
																delete_repo(token=cls._token   ,					repo_id='''valid_org/test-tokenizer-org'''   )
										except HTTPError:
																pass

										try:
																delete_repo(token=cls._token   ,					repo_id='''test-dynamic-tokenizer'''   )
										except HTTPError:
																pass




				def   a__(	self    :				Optional[Any]   )->  Any:

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					'''vocab.txt'''   )
																with open(lowerCAmelCase   ,					'''w'''   ,					encoding='''utf-8'''   ) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
																UpperCAmelCase         =			BertTokenizer(lowerCAmelCase   )

										tokenizer.push_to_hub('''test-tokenizer'''   ,					use_auth_token=self._token   )
										UpperCAmelCase         =			BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""   )
										self.assertDictEqual(new_tokenizer.vocab   ,					tokenizer.vocab   )

										# Reset repo
										delete_repo(token=self._token   ,					repo_id='''test-tokenizer'''   )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(lowerCAmelCase   ,					repo_id='''test-tokenizer'''   ,					push_to_hub=lowerCAmelCase   ,					use_auth_token=self._token   )

										UpperCAmelCase         =			BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""   )
										self.assertDictEqual(new_tokenizer.vocab   ,					tokenizer.vocab   )




				def   a__(	self    :				Union[str, Any]   )->  Optional[int]:

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					'''vocab.txt'''   )
																with open(lowerCAmelCase   ,					'''w'''   ,					encoding='''utf-8'''   ) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
																UpperCAmelCase         =			BertTokenizer(lowerCAmelCase   )

										tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''   ,					use_auth_token=self._token   )
										UpperCAmelCase         =			BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
										self.assertDictEqual(new_tokenizer.vocab   ,					tokenizer.vocab   )

										# Reset repo
										delete_repo(token=self._token   ,					repo_id='''valid_org/test-tokenizer-org'''   )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(
																    lowerCAmelCase   ,					repo_id='''valid_org/test-tokenizer-org'''   ,					push_to_hub=lowerCAmelCase   ,					use_auth_token=self._token   )

										UpperCAmelCase         =			BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
										self.assertDictEqual(new_tokenizer.vocab   ,					tokenizer.vocab   )





				@require_tokenizers
				def   a__(	self    :				int   )->  Dict:

										"""simple docstring"""




										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					'''vocab.txt'''   )
																with open(lowerCAmelCase   ,					'''w'''   ,					encoding='''utf-8'''   ) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
																UpperCAmelCase         =			CustomTokenizer(lowerCAmelCase   )

										# No fast custom tokenizer
										tokenizer.push_to_hub('''test-dynamic-tokenizer'''   ,					use_auth_token=self._token   )

										UpperCAmelCase         =			AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""   ,					trust_remote_code=lowerCAmelCase   )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__   ,					'''CustomTokenizer'''   )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
																UpperCAmelCase         =			os.path.join(lowerCAmelCase   ,					'''vocab.txt'''   )
																with open(lowerCAmelCase   ,					'''w'''   ,					encoding='''utf-8'''   ) as vocab_writer:
																						vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )

																UpperCAmelCase         =			BertTokenizerFast.from_pretrained(lowerCAmelCase   )
																bert_tokenizer.save_pretrained(lowerCAmelCase   )
																UpperCAmelCase         =			CustomTokenizerFast.from_pretrained(lowerCAmelCase   )

										tokenizer.push_to_hub('''test-dynamic-tokenizer'''   ,					use_auth_token=self._token   )

										UpperCAmelCase         =			AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""   ,					trust_remote_code=lowerCAmelCase   )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__   ,					'''CustomTokenizerFast'''   )
										UpperCAmelCase         =			AutoTokenizer.from_pretrained(
										    F"""{USER}/test-dynamic-tokenizer"""   ,					use_fast=lowerCAmelCase   ,					trust_remote_code=lowerCAmelCase   )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__   ,					'''CustomTokenizer'''   )







class  UpperCamelCase__(     unittest.TestCase ):




				def   a__(	self    :				int   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										trie.add('''Hello 友達'''   )
										self.assertEqual(trie.data   ,					{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )
										trie.add('''Hello'''   )
										trie.data
										self.assertEqual(trie.data   ,					{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )




				def   a__(	self    :				Union[str, Any]   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   )   ,					['''[CLS] This is a extra_id_100''']   )
										trie.add('''[CLS]'''   )
										trie.add('''extra_id_1'''   )
										trie.add('''extra_id_100'''   )
										self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   )   ,					['''[CLS]''', ''' This is a ''', '''extra_id_100''']   )




				def   a__(	self    :				Tuple   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										trie.add('''A'''   )
										self.assertEqual(trie.split('''ABC'''   )   ,					['''A''', '''BC''']   )
										self.assertEqual(trie.split('''BCA'''   )   ,					['''BC''', '''A''']   )




				def   a__(	self    :				Union[str, Any]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										trie.add('''TOKEN]'''   )
										trie.add('''[SPECIAL_TOKEN]'''   )
										self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   )   ,					['''This is something ''', '''[SPECIAL_TOKEN]''']   )




				def   a__(	self    :				Optional[Any]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										trie.add('''A'''   )
										trie.add('''P'''   )
										trie.add('''[SPECIAL_TOKEN]'''   )
										self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   )   ,					['''This is something ''', '''[SPECIAL_TOKEN]''']   )




				def   a__(	self    :				List[Any]   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										trie.add('''AB'''   )
										trie.add('''B'''   )
										trie.add('''C'''   )
										self.assertEqual(trie.split('''ABC'''   )   ,					['''AB''', '''C''']   )




				def   a__(	self    :				Union[str, Any]   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										trie.add('''ABC'''   )
										trie.add('''B'''   )
										trie.add('''CD'''   )
										self.assertEqual(trie.split('''ABCD'''   )   ,					['''ABC''', '''D''']   )





				def   a__(	self    :				Optional[int]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			Trie()
										UpperCAmelCase         =			trie.cut_text('''ABC'''   ,					[0, 0, 2, 1, 2, 3]   )
										self.assertEqual(lowerCAmelCase   ,					['''AB''', '''C''']   )









'''simple docstring'''



import time
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, torch_device

from ..test_modeling_common import ids_tensor


if is_torch_available():
	import torch

	from transformers.generation import (
	    MaxLengthCriteria,
	    MaxNewTokensCriteria,
	    MaxTimeCriteria,
	    StoppingCriteriaList,
	    validate_stopping_criteria,
	)






@require_torch
class  UpperCamelCase__(     unittest.TestCase ):




				def   a__(	self    :				List[Any]   ,					lowerCAmelCase    :				Optional[Any]   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			3
										UpperCAmelCase         =			250

										UpperCAmelCase         =			ids_tensor((batch_size, length)   ,					lowerCAmelCase   )
										UpperCAmelCase         =			torch.ones((batch_size, length)   ,					device=lowerCAmelCase   ,					dtype=torch.float   ) / length
										return input_ids, scores




				def   a__(	self    :				Dict   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(5   )

										UpperCAmelCase         =			StoppingCriteriaList(
										    [
										        MaxLengthCriteria(max_length=10   ),
										        MaxTimeCriteria(max_time=0.1   ),
										    ]   )

										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(9   )
										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(10   )
										self.assertTrue(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )




				def   a__(	self    :				Union[str, Any]   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			MaxLengthCriteria(max_length=10   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(5   )
										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(9   )
										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(10   )
										self.assertTrue(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )




				def   a__(	self    :				Optional[Any]   )->  List[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			MaxNewTokensCriteria(start_length=5   ,					max_new_tokens=5   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(5   )
										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(9   )
										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(10   )
										self.assertTrue(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase         =			StoppingCriteriaList([criteria]   )
										self.assertEqual(criteria_list.max_length   ,					10   )




				def   a__(	self    :				Tuple   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase       ,       UpperCAmelCase         =			self._get_tensors(5   )

										UpperCAmelCase         =			MaxTimeCriteria(max_time=0.1   )
										self.assertFalse(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )

										UpperCAmelCase         =			MaxTimeCriteria(max_time=0.1   ,					initial_timestamp=time.time() - 0.2   )
										self.assertTrue(criteria(lowerCAmelCase   ,					lowerCAmelCase   )   )





				def   a__(	self    :				int   )->  Any:

										"""simple docstring"""




										validate_stopping_criteria(StoppingCriteriaList([MaxLengthCriteria(10   )]   )   ,					10   )

										with self.assertWarns(lowerCAmelCase   ):
																validate_stopping_criteria(StoppingCriteriaList([MaxLengthCriteria(10   )]   )   ,					11   )

										UpperCAmelCase         =			validate_stopping_criteria(StoppingCriteriaList()   ,					11   )

										self.assertEqual(len(lowerCAmelCase   )   ,					1   )





