




import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
		from ...tokenization_utils_base import TextInput
from ...utils import logging


UpperCAmelCase_						=logging.get_logger(__name__)

UpperCAmelCase_						={"""vocab_file""": """spiece.model"""}

UpperCAmelCase_						={
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
UpperCAmelCase_						={
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}

UpperCAmelCase_						="""▁"""


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Tuple										=VOCAB_FILES_NAMES
				__a							: List[Any]										=PRETRAINED_VOCAB_FILES_MAP
				__a							: Any										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a							: Any										=["""input_ids""", """attention_mask"""]



				def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_="</s>"       , UpperCAmelCase_="<unk>"       , UpperCAmelCase_="<pad>"       , UpperCAmelCase_=1_00       , UpperCAmelCase_=None       , UpperCAmelCase_ = None       , UpperCAmelCase_=True       , **UpperCAmelCase_       , ):
											# Add extra_ids to the special token list
											if extra_ids > 0 and additional_special_tokens is None:
																		lowerCAmelCase   =		[F"""<extra_id_{i}>""" for i in range(UpperCAmelCase_						)]
											elif extra_ids > 0 and additional_special_tokens is not None:
																		# Check that we have the right number of extra_id special tokens
																		lowerCAmelCase   =		len(set(filter(lambda UpperCAmelCase_						: bool('''extra_id''' in str(UpperCAmelCase_						)						)       , UpperCAmelCase_						)						)						)
																		if extra_tokens != extra_ids:
																									raise ValueError(
																									    F"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
																									    ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
																									    ''' tokens'''						)
											if legacy:
																		logger.warning_once(
																		    F"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
																		    ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''						)

											lowerCAmelCase   =		legacy
											lowerCAmelCase   =		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    eos_token=UpperCAmelCase_       , unk_token=UpperCAmelCase_       , pad_token=UpperCAmelCase_       , extra_ids=UpperCAmelCase_       , additional_special_tokens=UpperCAmelCase_       , sp_model_kwargs=self.sp_model_kwargs       , legacy=UpperCAmelCase_       , **UpperCAmelCase_       , )

											lowerCAmelCase   =		vocab_file
											lowerCAmelCase   =		extra_ids

											lowerCAmelCase   =		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(UpperCAmelCase_						)



				@staticmethod
				def       __snake_case		( UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_						):
											if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
																		lowerCAmelCase   =		TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
																		if init_max_model_length is not None and init_max_model_length != max_model_length:
																									return init_max_model_length
																		elif init_max_model_length is None:
																									warnings.warn(
																									    '''This tokenizer was incorrectly instantiated with a model max length of'''
																									    F""" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"""
																									    ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
																									    ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
																									    F""" {pretrained_model_name_or_path} automatically truncating your input to"""
																									    F""" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"""
																									    F""" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"""
																									    ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
																									    ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       , UpperCAmelCase_       , )

											return max_model_length



				@property
				def       __snake_case		( self						):
											return self.sp_model.get_piece_size() + self._extra_ids



				def       __snake_case		( self						):
											lowerCAmelCase   =		{self.convert_ids_to_tokens(UpperCAmelCase_						): i for i in range(self.vocab_size						)}
											vocab.update(self.added_tokens_encoder						)
											return vocab



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None       , UpperCAmelCase_ = False						):
											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=UpperCAmelCase_       , token_ids_a=UpperCAmelCase_       , already_has_special_tokens=UpperCAmelCase_						)

											# normal case: some special tokens
											if token_ids_a is None:
																		return ([0] * len(UpperCAmelCase_						)) + [1]
											return ([0] * len(UpperCAmelCase_						)) + [1] + ([0] * len(UpperCAmelCase_						)) + [1]



				def       __snake_case		( self						):
											return list(
											    set(filter(lambda UpperCAmelCase_						: bool(re.search(r'''<extra_id_\d+>'''       , UpperCAmelCase_						)						) is not None       , self.additional_special_tokens						)						)						)



				def       __snake_case		( self						):
											return [self._convert_token_to_id(UpperCAmelCase_						) for token in self.get_sentinel_tokens()]



				def       __snake_case		( self       , UpperCAmelCase_						):
											if len(UpperCAmelCase_						) > 0 and token_ids[-1] == self.eos_token_id:
																		warnings.warn(
																		    F"""This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"""
																		    ''' eos tokens being added.'''						)
																		return token_ids
											else:
																		return token_ids + [self.eos_token_id]



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		[self.eos_token_id]

											if token_ids_a is None:
																		return len(token_ids_a + eos						) * [0]
											return len(token_ids_a + eos + token_ids_a + eos						) * [0]



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		self._add_eos_if_not_present(UpperCAmelCase_						)
											if token_ids_a is None:
																		return token_ids_a
											else:
																		lowerCAmelCase   =		self._add_eos_if_not_present(UpperCAmelCase_						)
																		return token_ids_a + token_ids_a



				def __getstate__( self						):
											lowerCAmelCase   =		self.__dict__.copy()
											lowerCAmelCase   =		None
											return state



				def __setstate__( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		d

											# for backward compatibility
											if not hasattr(self       , '''sp_model_kwargs'''						):
																		lowerCAmelCase   =		{}

											lowerCAmelCase   =		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(self.vocab_file						)



				def       __snake_case		( self       , UpperCAmelCase_       , **UpperCAmelCase_						):
											# Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
											# the beginning of the text
											if not self.legacy:
																		lowerCAmelCase   =		SPIECE_UNDERLINE + text.replace(UpperCAmelCase_       , ''' '''						)
											return super().tokenize(UpperCAmelCase_       , **UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_       , **UpperCAmelCase_						):
											if not self.legacy:
																		lowerCAmelCase   =		text.startswith(UpperCAmelCase_						)
																		if is_first:
																									lowerCAmelCase   =		text[1:]

											lowerCAmelCase   =		self.sp_model.encode(UpperCAmelCase_       , out_type=UpperCAmelCase_						)

											if not self.legacy and not is_first and not text.startswith(''' '''						) and tokens[0].startswith(UpperCAmelCase_						):
																		lowerCAmelCase   =		([tokens[0][1:]] if len(tokens[0]						) > 1 else []) + tokens[1:]
											return tokens



				def       __snake_case		( self       , UpperCAmelCase_						):
											if token.startswith('''<extra_id_'''						):
																		lowerCAmelCase   =		re.match(r'''<extra_id_(\d+)>'''       , UpperCAmelCase_						)
																		lowerCAmelCase   =		int(match.group(1						)						)
																		return self.vocab_size - num - 1
											return self.sp_model.piece_to_id(UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											if index < self.sp_model.get_piece_size():
																		lowerCAmelCase   =		self.sp_model.IdToPiece(UpperCAmelCase_						)
											else:
																		lowerCAmelCase   =		F"""<extra_id_{self.vocab_size - 1 - index}>"""
											return token



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		[]
											lowerCAmelCase   =		''''''
											lowerCAmelCase   =		False
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									if not prev_is_special:
																																out_string += " "
																									out_string += self.sp_model.decode(UpperCAmelCase_						) + token
																									lowerCAmelCase   =		True
																									lowerCAmelCase   =		[]
																		else:
																									current_sub_tokens.append(UpperCAmelCase_						)
																									lowerCAmelCase   =		False
											out_string += self.sp_model.decode(UpperCAmelCase_						)
											return out_string.strip()






				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											if not os.path.isdir(UpperCAmelCase_						):
																		logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
																		return
											lowerCAmelCase   =		os.path.join(
											    UpperCAmelCase_       , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

											if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase_						) and os.path.isfile(self.vocab_file						):
																		copyfile(self.vocab_file       , UpperCAmelCase_						)
											elif not os.path.isfile(self.vocab_file						):
																		with open(UpperCAmelCase_       , '''wb'''						) as fi:
																									lowerCAmelCase   =		self.sp_model.serialized_model_proto()
																									fi.write(UpperCAmelCase_						)

											return (out_vocab_file,)









import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


UpperCAmelCase_						="""hf-internal-testing/tiny-random-bert"""
UpperCAmelCase_						=os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
UpperCAmelCase_						="""9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""


class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self						):
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_						)
											# Should have downloaded the file in here
											self.assertTrue(os.path.isdir(UpperCAmelCase_						)						)
											# Cache should contain at least those three subfolders:
											for subfolder in ["blobs", "refs", "snapshots"]:
																		self.assertTrue(os.path.isdir(os.path.join(UpperCAmelCase_       , UpperCAmelCase_						)						)						)
											with open(os.path.join(UpperCAmelCase_       , '''refs'''       , '''main'''						)						) as f:
																		lowerCAmelCase   =		f.read()
											self.assertEqual(UpperCAmelCase_       , os.path.join(UpperCAmelCase_       , '''snapshots'''       , UpperCAmelCase_       , UpperCAmelCase_						)						)
											self.assertTrue(os.path.isfile(UpperCAmelCase_						)						)

											# File is cached at the same place the second time.
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_						)
											self.assertEqual(UpperCAmelCase_       , UpperCAmelCase_						)

											# Using a specific revision to test the full commit hash.
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_       , revision='''9b8c223'''						)
											self.assertEqual(UpperCAmelCase_       , os.path.join(UpperCAmelCase_       , '''snapshots'''       , UpperCAmelCase_       , UpperCAmelCase_						)						)



				def       __snake_case		( self						):
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid model identifier'''						):
																		lowerCAmelCase   =		cached_file('''tiny-random-bert'''       , UpperCAmelCase_						)

											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid git identifier'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_       , revision='''aaaa'''						)

											with self.assertRaisesRegex(UpperCAmelCase_       , '''does not appear to have a file named'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''						)



				def       __snake_case		( self						):
											with self.assertRaisesRegex(UpperCAmelCase_       , '''does not appear to have a file named'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''						)

											with open(os.path.join(UpperCAmelCase_       , '''refs'''       , '''main'''						)						) as f:
																		lowerCAmelCase   =		f.read()
											self.assertTrue(os.path.isfile(os.path.join(UpperCAmelCase_       , '''.no_exist'''       , UpperCAmelCase_       , '''conf'''						)						)						)

											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , _raise_exceptions_for_missing_entries=UpperCAmelCase_						)
											self.assertIsNone(UpperCAmelCase_						)

											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , local_files_only=UpperCAmelCase_       , _raise_exceptions_for_missing_entries=UpperCAmelCase_						)
											self.assertIsNone(UpperCAmelCase_						)

											lowerCAmelCase   =		mock.Mock()
											lowerCAmelCase   =		5_00
											lowerCAmelCase   =		{}
											lowerCAmelCase   =		HTTPError
											lowerCAmelCase   =		{}

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('''requests.Session.request'''       , return_value=UpperCAmelCase_						) as mock_head:
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , _raise_exceptions_for_connection_errors=UpperCAmelCase_						)
																		self.assertIsNone(UpperCAmelCase_						)
																		# This check we did call the fake head request
																		mock_head.assert_called()



				def       __snake_case		( self						):
											self.assertTrue(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)
											self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)
											self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)



				def       __snake_case		( self						):
											# `get_file_from_repo` returns None if the file does not exist
											self.assertIsNone(get_file_from_repo('''bert-base-cased'''       , '''ahah.txt'''						)						)

											# The function raises if the repository does not exist.
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid model identifier'''						):
																		get_file_from_repo('''bert-base-case'''       , UpperCAmelCase_						)

											# The function raises if the revision does not exist.
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid git identifier'''						):
																		get_file_from_repo('''bert-base-cased'''       , UpperCAmelCase_       , revision='''ahaha'''						)

											lowerCAmelCase   =		get_file_from_repo('''bert-base-cased'''       , UpperCAmelCase_						)
											# The name is the cached name which is not very easy to test, so instead we load the content.
											lowerCAmelCase   =		json.loads(open(UpperCAmelCase_       , '''r'''						).read()						)
											self.assertEqual(config['''hidden_size''']       , 7_68						)






				def       __snake_case		( self						):
											with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase   =		Path(UpperCAmelCase_						) / '''a.txt'''
																		filename.touch()
																		self.assertEqual(get_file_from_repo(UpperCAmelCase_       , '''a.txt'''						)       , str(UpperCAmelCase_						)						)

																		self.assertIsNone(get_file_from_repo(UpperCAmelCase_       , '''b.txt'''						)						)



