



"""simple docstring"""





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase      =		logging.get_logger(__name__)

lowerCamelCase      =		{"""vocab_file""": """spiece.model"""}

lowerCamelCase      =		{
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    }
}
class    lowercase__     (					SCREAMING_SNAKE_CASE   ):







       '''simple docstring'''


       def __init__(							self    :						List[Any]	,       _UpperCAmelCase    :						int	,       _UpperCAmelCase    :						int=False	,       _UpperCAmelCase    :						Optional[int]=True	,       _UpperCAmelCase    :						str=False	,       _UpperCAmelCase    :						Tuple="<s>"	,       _UpperCAmelCase    :						Tuple="</s>"	,       _UpperCAmelCase    :						Optional[Any]="<unk>"	,       _UpperCAmelCase    :						List[str]="<sep>"	,       _UpperCAmelCase    :						Union[str, Any]="<pad>"	,       _UpperCAmelCase    :						int="<cls>"	,       _UpperCAmelCase    :						int="<mask>"	,       _UpperCAmelCase    :						Any=["<eop>", "<eod>"]	,       _UpperCAmelCase    :						Optional[Dict[str, Any]] = None	,       **_UpperCAmelCase    :						Dict	,       )				->		None:


             '''simple docstring'''




             UpperCAmelCase_    				=							AddedToken(_UpperCAmelCase	,       lstrip=_UpperCAmelCase	,       rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase	,       _UpperCAmelCase					) else mask_token

             UpperCAmelCase_    				=							{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=_UpperCAmelCase	,       remove_space=_UpperCAmelCase	,       keep_accents=_UpperCAmelCase	,       bos_token=_UpperCAmelCase	,       eos_token=_UpperCAmelCase	,       unk_token=_UpperCAmelCase	,       sep_token=_UpperCAmelCase	,       pad_token=_UpperCAmelCase	,       cls_token=_UpperCAmelCase	,       mask_token=_UpperCAmelCase	,       additional_special_tokens=_UpperCAmelCase	,       sp_model_kwargs=self.sp_model_kwargs	,       **_UpperCAmelCase	,       )

             UpperCAmelCase_    				=							3

             UpperCAmelCase_    				=							do_lower_case
             UpperCAmelCase_    				=							remove_space
             UpperCAmelCase_    				=							keep_accents
             UpperCAmelCase_    				=							vocab_file

             UpperCAmelCase_    				=							spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(_UpperCAmelCase					)

             try:
                   import jieba
             except ModuleNotFoundError as error:
                   raise error.__class__(
                       "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                       "See https://pypi.org/project/jieba/ for installation."					)
             UpperCAmelCase_    				=							jieba
             UpperCAmelCase_    				=							str.maketrans(" \n"	,       "\u2582\u2583"					)



       @property
       # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
       def   lowercase__       (							self    :						Optional[int]					)				->		str:


             '''simple docstring'''




             return len(self.sp_model					)



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							{self.convert_ids_to_tokens(_UpperCAmelCase					): i for i in range(self.vocab_size					)}
             vocab.update(self.added_tokens_encoder					)
             return vocab



       def __getstate__(							self    :						str					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.__dict__.copy()
             UpperCAmelCase_    				=							None
             return state



       def __setstate__(							self    :						str	,       _UpperCAmelCase    :						Optional[int]					)				->		Dict:


             '''simple docstring'''




             UpperCAmelCase_    				=							d

             # for backward compatibility
             if not hasattr(self	,       "sp_model_kwargs"					):
                   UpperCAmelCase_    				=							{}

             UpperCAmelCase_    				=							spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(self.vocab_file					)



       def   lowercase__       (							self    :						Any	,       _UpperCAmelCase    :						Optional[Any]					)				->		str:


             '''simple docstring'''




             if self.remove_space:
                   UpperCAmelCase_    				=							" ".join(inputs.strip().split()					)
             else:
                   UpperCAmelCase_    				=							inputs
             UpperCAmelCase_    				=							outputs.replace("``"	,       "\""					).replace("''"	,       "\""					)

             if not self.keep_accents:
                   UpperCAmelCase_    				=							unicodedata.normalize("NFKD"	,       _UpperCAmelCase					)
                   UpperCAmelCase_    				=							"".join([c for c in outputs if not unicodedata.combining(_UpperCAmelCase					)]					)
             if self.do_lower_case:
                   UpperCAmelCase_    				=							outputs.lower()

             return outputs



       def   lowercase__       (							self    :						Dict	,       _UpperCAmelCase    :						str					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.preprocess_text(_UpperCAmelCase					)
             UpperCAmelCase_    				=							self.sp_model.encode(_UpperCAmelCase	,       out_type=_UpperCAmelCase					)
             UpperCAmelCase_    				=							[]
             for piece in pieces:
                   if len(_UpperCAmelCase					) > 1 and piece[-1] == str(","					) and piece[-2].isdigit():
                         UpperCAmelCase_    				=							self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCAmelCase	,       ""					)					)
                         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                               if len(cur_pieces[0]					) == 1:
                                     UpperCAmelCase_    				=							cur_pieces[1:]
                               else:
                                     UpperCAmelCase_    				=							cur_pieces[0][1:]
                         cur_pieces.append(piece[-1]					)
                         new_pieces.extend(_UpperCAmelCase					)
                   else:
                         new_pieces.append(_UpperCAmelCase					)

             return new_pieces



       def   lowercase__       (							self    :						List[Any]	,       _UpperCAmelCase    :						List[Any]					)				->		Union[str, Any]:


             '''simple docstring'''




             return self.sp_model.PieceToId(_UpperCAmelCase					)



       def   lowercase__       (							self    :						str	,       _UpperCAmelCase    :						Any					)				->		List[Any]:


             '''simple docstring'''




             return self.sp_model.IdToPiece(_UpperCAmelCase					)



       def   lowercase__       (							self    :						List[Any]	,       _UpperCAmelCase    :						Dict					)				->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							"".join(_UpperCAmelCase					).replace(_UpperCAmelCase	,       " "					).strip()
             return out_string



       def   lowercase__       (							self    :						Any	,       _UpperCAmelCase    :						List[int]	,       _UpperCAmelCase    :						Optional[List[int]] = None					)				->		List[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							[self.sep_token_id]
             UpperCAmelCase_    				=							[self.cls_token_id]
             if token_ids_a is None:
                   return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls



       def   lowercase__       (							self    :						List[Any]	,       _UpperCAmelCase    :						List[int]	,       _UpperCAmelCase    :						Optional[List[int]] = None	,       _UpperCAmelCase    :						bool = False					)				->		List[int]:


             '''simple docstring'''




             if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=_UpperCAmelCase	,       token_ids_a=_UpperCAmelCase	,       already_has_special_tokens=_UpperCAmelCase					)

             if token_ids_a is not None:
                   return ([0] * len(_UpperCAmelCase					)) + [1] + ([0] * len(_UpperCAmelCase					)) + [1, 1]
             return ([0] * len(_UpperCAmelCase					)) + [1, 1]



       def   lowercase__       (							self    :						Tuple	,       _UpperCAmelCase    :						List[int]	,       _UpperCAmelCase    :						Optional[List[int]] = None					)				->		List[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							[self.sep_token_id]
             UpperCAmelCase_    				=							[2]

             if token_ids_a is None:
                   return len(token_ids_a + sep					) * [0] + cls_segment_id
             return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1] + cls_segment_id



       def   lowercase__       (							self    :						Optional[Any]	,       _UpperCAmelCase    :						str	,       _UpperCAmelCase    :						Optional[str] = None					)				->		Tuple[str]:


             '''simple docstring'''




             if not os.path.isdir(_UpperCAmelCase					):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
                   return
             UpperCAmelCase_    				=							os.path.join(
                 _UpperCAmelCase	,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)

             if os.path.abspath(self.vocab_file					) != os.path.abspath(_UpperCAmelCase					) and os.path.isfile(self.vocab_file					):
                   copyfile(self.vocab_file	,       _UpperCAmelCase					)
             elif not os.path.isfile(self.vocab_file					):
                   with open(_UpperCAmelCase	,       "wb"					) as fi:
                         UpperCAmelCase_    				=							self.sp_model.serialized_model_proto()
                         fi.write(_UpperCAmelCase					)

             return (out_vocab_file,)







       def   lowercase__       (							self    :						Union[str, Any]	,       *_UpperCAmelCase    :						Optional[int]	,       **_UpperCAmelCase    :						List[str]					)				->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							super()._decode(*_UpperCAmelCase	,       **_UpperCAmelCase					)
             UpperCAmelCase_    				=							text.replace(" "	,       ""					).replace("\u2582"	,       " "					).replace("\u2583"	,       "\n"					)
             return text









"""simple docstring"""





import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      =		get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
class    lowercase__     (					SCREAMING_SNAKE_CASE       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  XLMProphetNetTokenizer
       UpperCamelCase		=  False
       UpperCamelCase		=  True



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Any:


             '''simple docstring'''




             super().setUp()

             # We have a SentencePiece fixture for testing
             UpperCAmelCase_    				=							XLMProphetNetTokenizer(_UpperCAmelCase	,       keep_accents=_UpperCAmelCase					)
             tokenizer.save_pretrained(self.tmpdirname					)



       def   lowercase__       (							self    :						Tuple					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							"[PAD]"
             UpperCAmelCase_    				=							0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase					)	,       _UpperCAmelCase					)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase					)	,       _UpperCAmelCase					)



       def   lowercase__       (							self    :						Dict					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							list(self.get_tokenizer().get_vocab().keys()					)

             self.assertEqual(vocab_keys[0]	,       "[PAD]"					)
             self.assertEqual(vocab_keys[1]	,       "[CLS]"					)
             self.assertEqual(vocab_keys[-1]	,       "j"					)
             self.assertEqual(len(_UpperCAmelCase					)	,       1012					)



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             self.assertEqual(self.get_tokenizer().vocab_size	,       1012					)



       def   lowercase__       (							self    :						Tuple					)				->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							XLMProphetNetTokenizer(_UpperCAmelCase	,       keep_accents=_UpperCAmelCase					)

             UpperCAmelCase_    				=							tokenizer.tokenize("This is a test"					)
             self.assertListEqual(_UpperCAmelCase	,       ["▁This", "▁is", "▁a", "▁t", "est"]					)

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)	,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,       )

             UpperCAmelCase_    				=							tokenizer.tokenize("I was born in 92000, and this is falsé."					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "9",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "é",
                     ".",
                 ]	,       )
             UpperCAmelCase_    				=							tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
                 ]	,       )

             UpperCAmelCase_    				=							tokenizer.convert_ids_to_tokens(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "[UNK]",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "[UNK]",
                     ".",
                 ]	,       )



       @cached_property
       def   lowercase__       (							self    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"					)



       @slow
       def   lowercase__       (							self    :						List[Any]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							"Hello World!"
             UpperCAmelCase_    				=							[35389, 6672, 49, 2]
             self.assertListEqual(_UpperCAmelCase	,       self.big_tokenizer.encode(_UpperCAmelCase					)					)







       @slow
       def   lowercase__       (							self    :						Optional[int]					)				->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							{"input_ids": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=_UpperCAmelCase	,       model_name="microsoft/xprophetnet-large-wiki100-cased"	,       revision="1acad1643ddd54a44df6a1b797ada8373685d90e"	,       )




