


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A							:					Optional[int]     						=     logging.get_logger(__name__)
A							:					List[Any]     						=     {'vocab_file': 'spiece.model'}

A							:					int     						=     {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

A							:					Optional[Any]     						=     {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

A							:					List[Any]     						=     '▁'






class     UpperCamelCase(  _a   ):
     snake_case_		:				Union[str, Any]										=	VOCAB_FILES_NAMES
     snake_case_		:				Dict										=	PRETRAINED_VOCAB_FILES_MAP
     snake_case_		:				Any										=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


     def __init__(						self	:		Tuple     ,    SCREAMING_SNAKE_CASE	:		Optional[int]     ,    SCREAMING_SNAKE_CASE	:		Tuple=True     ,    SCREAMING_SNAKE_CASE	:		List[Any]=True     ,    SCREAMING_SNAKE_CASE	:		List[str]=False     ,    SCREAMING_SNAKE_CASE	:		int="[CLS]"     ,    SCREAMING_SNAKE_CASE	:		int="[SEP]"     ,    SCREAMING_SNAKE_CASE	:		Union[str, Any]="<unk>"     ,    SCREAMING_SNAKE_CASE	:		Any="[SEP]"     ,    SCREAMING_SNAKE_CASE	:		List[Any]="<pad>"     ,    SCREAMING_SNAKE_CASE	:		Dict="[CLS]"     ,    SCREAMING_SNAKE_CASE	:		List[Any]="[MASK]"     ,    SCREAMING_SNAKE_CASE	:		Optional[Dict[str, Any]] = None     ,    **SCREAMING_SNAKE_CASE	:		Tuple     ,    )			->	None:




           '''simple docstring'''

           __snake_case				=		(
               AddedToken(SCREAMING_SNAKE_CASE     ,    lstrip=SCREAMING_SNAKE_CASE     ,    rstrip=SCREAMING_SNAKE_CASE     ,    normalized=SCREAMING_SNAKE_CASE						)
               if isinstance(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)
               else mask_token
           )

           __snake_case				=		{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=SCREAMING_SNAKE_CASE     ,    remove_space=SCREAMING_SNAKE_CASE     ,    keep_accents=SCREAMING_SNAKE_CASE     ,    bos_token=SCREAMING_SNAKE_CASE     ,    eos_token=SCREAMING_SNAKE_CASE     ,    unk_token=SCREAMING_SNAKE_CASE     ,    sep_token=SCREAMING_SNAKE_CASE     ,    pad_token=SCREAMING_SNAKE_CASE     ,    cls_token=SCREAMING_SNAKE_CASE     ,    mask_token=SCREAMING_SNAKE_CASE     ,    sp_model_kwargs=self.sp_model_kwargs     ,    **SCREAMING_SNAKE_CASE     ,    )

           __snake_case				=		do_lower_case
           __snake_case				=		remove_space
           __snake_case				=		keep_accents
           __snake_case				=		vocab_file

           __snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
           self.sp_model.Load(SCREAMING_SNAKE_CASE						)


     @property
     def 							SCREAMING_SNAKE_CASE_  (						self	:		List[str]						)			->	Optional[Any]:




           '''simple docstring'''

           return len(self.sp_model						)


     def 							SCREAMING_SNAKE_CASE_  (						self	:		Any						)			->	Dict:




           '''simple docstring'''

           __snake_case				=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE						): i for i in range(self.vocab_size						)}
           vocab.update(self.added_tokens_encoder						)
           return vocab


     def __getstate__(						self	:		Dict						)			->	int:




           '''simple docstring'''

           __snake_case				=		self.__dict__.copy()
           __snake_case				=		None
           return state


     def __setstate__(						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		Dict						)			->	Any:




           '''simple docstring'''

           __snake_case				=		d

           # for backward compatibility
           if not hasattr(self     ,    "sp_model_kwargs"						):
                 __snake_case				=		{}

           __snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs						)
           self.sp_model.Load(self.vocab_file						)


     def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple     ,    SCREAMING_SNAKE_CASE	:		Optional[int]						)			->	Any:




           '''simple docstring'''

           if self.remove_space:
                 __snake_case				=		" ".join(inputs.strip().split()						)
           else:
                 __snake_case				=		inputs
           __snake_case				=		outputs.replace("``"     ,    "\""						).replace("''"     ,    "\""						)

           if not self.keep_accents:
                 __snake_case				=		unicodedata.normalize("NFKD"     ,    SCREAMING_SNAKE_CASE						)
                 __snake_case				=		"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE						)]						)
           if self.do_lower_case:
                 __snake_case				=		outputs.lower()

           return outputs


     def 							SCREAMING_SNAKE_CASE_  (						self	:		int     ,    SCREAMING_SNAKE_CASE	:		str						)			->	List[str]:




           '''simple docstring'''

           __snake_case				=		self.preprocess_text(SCREAMING_SNAKE_CASE						)
           __snake_case				=		self.sp_model.encode(SCREAMING_SNAKE_CASE     ,    out_type=SCREAMING_SNAKE_CASE						)
           __snake_case				=		[]
           for piece in pieces:
                 if len(SCREAMING_SNAKE_CASE						) > 1 and piece[-1] == str(","						) and piece[-2].isdigit():
                       __snake_case				=		self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE     ,    ""						)						)
                       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                             if len(cur_pieces[0]						) == 1:
                                   __snake_case				=		cur_pieces[1:]
                             else:
                                   __snake_case				=		cur_pieces[0][1:]
                       cur_pieces.append(piece[-1]						)
                       new_pieces.extend(SCREAMING_SNAKE_CASE						)
                 else:
                       new_pieces.append(SCREAMING_SNAKE_CASE						)

           return new_pieces


     def 							SCREAMING_SNAKE_CASE_  (						self	:		str     ,    SCREAMING_SNAKE_CASE	:		Optional[Any]						)			->	int:




           '''simple docstring'''

           return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE						)


     def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]     ,    SCREAMING_SNAKE_CASE	:		Tuple						)			->	Optional[Any]:




           '''simple docstring'''

           return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE						)


     def 							SCREAMING_SNAKE_CASE_  (						self	:		int     ,    SCREAMING_SNAKE_CASE	:		Union[str, Any]						)			->	Dict:




           '''simple docstring'''

           __snake_case				=		[]
           __snake_case				=		""
           __snake_case				=		False
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       if not prev_is_special:
                             out_string += " "
                       out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE						) + token
                       __snake_case				=		True
                       __snake_case				=		[]
                 else:
                       current_sub_tokens.append(SCREAMING_SNAKE_CASE						)
                       __snake_case				=		False
           out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE						)
           return out_string.strip()


     def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict     ,    SCREAMING_SNAKE_CASE	:		List[int]     ,    SCREAMING_SNAKE_CASE	:		Optional[List[int]] = None						)			->	List[int]:




           '''simple docstring'''

           __snake_case				=		[self.sep_token_id]
           __snake_case				=		[self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep


     def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict     ,    SCREAMING_SNAKE_CASE	:		List[int]     ,    SCREAMING_SNAKE_CASE	:		Optional[List[int]] = None     ,    SCREAMING_SNAKE_CASE	:		bool = False						)			->	List[int]:




           '''simple docstring'''

           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=SCREAMING_SNAKE_CASE     ,    token_ids_a=SCREAMING_SNAKE_CASE     ,    already_has_special_tokens=SCREAMING_SNAKE_CASE						)

           if token_ids_a is not None:
                 return [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1]
           return [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1]


     def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		List[int]     ,    SCREAMING_SNAKE_CASE	:		Optional[List[int]] = None						)			->	List[int]:




           '''simple docstring'''

           __snake_case				=		[self.sep_token_id]
           __snake_case				=		[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep						) * [0]
           return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






     def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]     ,    SCREAMING_SNAKE_CASE	:		str     ,    SCREAMING_SNAKE_CASE	:		Optional[str] = None						)			->	Tuple[str]:




           '''simple docstring'''

           if not os.path.isdir(SCREAMING_SNAKE_CASE						):
                 logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
                 return
           __snake_case				=		os.path.join(
               SCREAMING_SNAKE_CASE     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

           if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE						) and os.path.isfile(self.vocab_file						):
                 copyfile(self.vocab_file     ,    SCREAMING_SNAKE_CASE						)
           elif not os.path.isfile(self.vocab_file						):
                 with open(SCREAMING_SNAKE_CASE     ,    "wb"						) as fi:
                       __snake_case				=		self.sp_model.serialized_model_proto()
                       fi.write(SCREAMING_SNAKE_CASE						)

           return (out_vocab_file,)









import os
import tempfile
import unittest
import uuid
from pathlib import Path

from transformers.testing_utils import get_tests_dir, require_soundfile, require_torch, require_vision
from transformers.tools.agent_types import AgentAudio, AgentImage, AgentText
from transformers.utils import is_soundfile_availble, is_torch_available, is_vision_available


if is_torch_available():
	import torch

if is_soundfile_availble():
	import soundfile as sf

if is_vision_available():
	from PIL import Image





def 	_lowerCAmelCase       (    _lowerCAmelCase="" )						->	str:


						'''simple docstring'''






						__snake_case				=		tempfile.mkdtemp()
						return os.path.join(_lowerCAmelCase      ,	str(uuid.uuida() ) + suffix )






@require_soundfile
@require_torch
class     UpperCamelCase(  unittest.TestCase   ):


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]						)			->	Optional[int]:




											'''simple docstring'''

											__snake_case				=		torch.rand(1_2     ,    dtype=torch.floataa						) - 0.5
											__snake_case				=		AgentAudio(SCREAMING_SNAKE_CASE						)
											__snake_case				=		str(agent_type.to_string()						)

											# Ensure that the tensor and the agent_type's tensor are the same
											self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE     ,    agent_type.to_raw()     ,    atol=1e-4						)						)

											del agent_type

											# Ensure the path remains even after the object deletion
											self.assertTrue(os.path.exists(SCREAMING_SNAKE_CASE						)						)

											# Ensure that the file contains the same value as the original tensor
											__snake_case		,					__snake_case				=		sf.read(SCREAMING_SNAKE_CASE						)
											self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE     ,    torch.tensor(SCREAMING_SNAKE_CASE						)     ,    atol=1e-4						)						)






					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict						)			->	int:




											'''simple docstring'''

											__snake_case				=		torch.rand(1_2     ,    dtype=torch.floataa						) - 0.5
											__snake_case				=		get_new_path(suffix=".wav"						)
											sf.write(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE     ,    1_6_0_0_0						)

											__snake_case				=		AgentAudio(SCREAMING_SNAKE_CASE						)

											self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE     ,    agent_type.to_raw()     ,    atol=1e-4						)						)
											self.assertEqual(agent_type.to_string()     ,    SCREAMING_SNAKE_CASE						)






@require_vision
@require_torch
class     UpperCamelCase(  unittest.TestCase   ):


					def 							SCREAMING_SNAKE_CASE_  (						self	:		str						)			->	Optional[int]:




											'''simple docstring'''

											__snake_case				=		torch.randint(0     ,    2_5_6     ,    (6_4, 6_4, 3)						)
											__snake_case				=		AgentImage(SCREAMING_SNAKE_CASE						)
											__snake_case				=		str(agent_type.to_string()						)

											# Ensure that the tensor and the agent_type's tensor are the same
											self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE     ,    agent_type._tensor     ,    atol=1e-4						)						)

											self.assertIsInstance(agent_type.to_raw()     ,    Image.Image						)

											# Ensure the path remains even after the object deletion
											del agent_type
											self.assertTrue(os.path.exists(SCREAMING_SNAKE_CASE						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple						)			->	Any:




											'''simple docstring'''

											__snake_case				=		Path(get_tests_dir("fixtures/tests_samples/COCO"						)						) / "000000039769.png"
											__snake_case				=		Image.open(SCREAMING_SNAKE_CASE						)
											__snake_case				=		AgentImage(SCREAMING_SNAKE_CASE						)

											self.assertTrue(path.samefile(agent_type.to_string()						)						)
											self.assertTrue(image == agent_type.to_raw()						)

											# Ensure the path remains even after the object deletion
											del agent_type
											self.assertTrue(os.path.exists(SCREAMING_SNAKE_CASE						)						)






					def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]						)			->	Union[str, Any]:




											'''simple docstring'''

											__snake_case				=		Path(get_tests_dir("fixtures/tests_samples/COCO"						)						) / "000000039769.png"
											__snake_case				=		Image.open(SCREAMING_SNAKE_CASE						)
											__snake_case				=		AgentImage(SCREAMING_SNAKE_CASE						)

											self.assertFalse(path.samefile(agent_type.to_string()						)						)
											self.assertTrue(image == agent_type.to_raw()						)

											# Ensure the path remains even after the object deletion
											del agent_type
											self.assertTrue(os.path.exists(SCREAMING_SNAKE_CASE						)						)






class     UpperCamelCase(  unittest.TestCase   ):


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Any						)			->	Any:




											'''simple docstring'''

											__snake_case				=		"Hey!"
											__snake_case				=		AgentText(SCREAMING_SNAKE_CASE						)

											self.assertEqual(SCREAMING_SNAKE_CASE     ,    agent_type.to_string()						)
											self.assertEqual(SCREAMING_SNAKE_CASE     ,    agent_type.to_raw()						)
											self.assertEqual(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE						)





