



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a_						:     int	     =		logging.get_logger(__name__)

a_						:     List[str]	     =		{'vocab_file': 'spiece.model'}

a_						:     Optional[Any]	     =		{
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     def __init__(self				,			__a				,			__a=False				,			__a=True				,			__a=False				,			__a="<s>"				,			__a="</s>"				,			__a="<unk>"				,			__a="<sep>"				,			__a="<pad>"				,			__a="<cls>"				,			__a="<mask>"				,			__a=["<eop>", "<eod>"]				,			__a = None				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		AddedToken(__a				,			lstrip=__a				,			rstrip=__a    ) if isinstance(__a				,			__a    ) else mask_token

           lowerCamelCase								=		{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=__a				,			remove_space=__a				,			keep_accents=__a				,			bos_token=__a				,			eos_token=__a				,			unk_token=__a				,			sep_token=__a				,			pad_token=__a				,			cls_token=__a				,			mask_token=__a				,			additional_special_tokens=__a				,			sp_model_kwargs=self.sp_model_kwargs				,			**__a				,			)

           lowerCamelCase								=		3

           lowerCamelCase								=		do_lower_case
           lowerCamelCase								=		remove_space
           lowerCamelCase								=		keep_accents
           lowerCamelCase								=		vocab_file

           lowerCamelCase								=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
           self.sp_model.Load(__a    )

           try:
                 import jieba
           except ModuleNotFoundError as error:
                 raise error.__class__(
                     "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                     "See https://pypi.org/project/jieba/ for installation."    )
           lowerCamelCase								=		jieba
           lowerCamelCase								=		str.maketrans(" \n"				,			"\u2582\u2583"    )







     @property
     # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
     def 						_a      (self    ):






           '''simple docstring'''



           return len(self.sp_model    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		{self.convert_ids_to_tokens(__a    ): i for i in range(self.vocab_size    )}
           vocab.update(self.added_tokens_encoder    )
           return vocab







     def __getstate__(self    ):






           '''simple docstring'''



           lowerCamelCase								=		self.__dict__.copy()
           lowerCamelCase								=		None
           return state







     def __setstate__(self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		d

           # for backward compatibility
           if not hasattr(self				,			"sp_model_kwargs"    ):
                 lowerCamelCase								=		{}

           lowerCamelCase								=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
           self.sp_model.Load(self.vocab_file    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           if self.remove_space:
                 lowerCamelCase								=		" ".join(inputs.strip().split()    )
           else:
                 lowerCamelCase								=		inputs
           lowerCamelCase								=		outputs.replace("``"				,			"\""    ).replace("''"				,			"\""    )

           if not self.keep_accents:
                 lowerCamelCase								=		unicodedata.normalize("NFKD"				,			__a    )
                 lowerCamelCase								=		"".join([c for c in outputs if not unicodedata.combining(__a    )]    )
           if self.do_lower_case:
                 lowerCamelCase								=		outputs.lower()

           return outputs







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.preprocess_text(__a    )
           lowerCamelCase								=		self.sp_model.encode(__a				,			out_type=__a    )
           lowerCamelCase								=		[]
           for piece in pieces:
                 if len(__a    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
                       lowerCamelCase								=		self.sp_model.EncodeAsPieces(piece[:-1].replace(__a				,			""    )    )
                       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                             if len(cur_pieces[0]    ) == 1:
                                   lowerCamelCase								=		cur_pieces[1:]
                             else:
                                   lowerCamelCase								=		cur_pieces[0][1:]
                       cur_pieces.append(piece[-1]    )
                       new_pieces.extend(__a    )
                 else:
                       new_pieces.append(__a    )

           return new_pieces







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.PieceToId(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.IdToPiece(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		"".join(__a    ).replace(__a				,			" "    ).strip()
           return out_string







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]
           if token_ids_a is None:
                 return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls







     def 						_a      (self				,			__a				,			__a = None				,			__a = False    ):






           '''simple docstring'''



           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__a				,			token_ids_a=__a				,			already_has_special_tokens=__a    )

           if token_ids_a is not None:
                 return ([0] * len(__a    )) + [1] + ([0] * len(__a    )) + [1, 1]
           return ([0] * len(__a    )) + [1, 1]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[2]

           if token_ids_a is None:
                 return len(token_ids_a + sep    ) * [0] + cls_segment_id
           return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           if not os.path.isdir(__a    ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
                 return
           lowerCamelCase								=		os.path.join(
               __a				,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

           if os.path.abspath(self.vocab_file    ) != os.path.abspath(__a    ) and os.path.isfile(self.vocab_file    ):
                 copyfile(self.vocab_file				,			__a    )
           elif not os.path.isfile(self.vocab_file    ):
                 with open(__a				,			"wb"    ) as fi:
                       lowerCamelCase								=		self.sp_model.serialized_model_proto()
                       fi.write(__a    )

           return (out_vocab_file,)







     def 						_a      (self				,			*__a				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase								=		super()._decode(*__a				,			**__a    )
           lowerCamelCase								=		text.replace(" "				,			""    ).replace("\u2582"				,			" "    ).replace("\u2583"				,			"\n"    )
           return text




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast





class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		mock.Mock()
           lowerCamelCase								=		5_00
           lowerCamelCase								=		{}
           lowerCamelCase								=		HTTPError
           lowerCamelCase								=		{}

           # Download this model to make sure it's in the cache.
           lowerCamelCase								=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"    )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("requests.Session.request"				,			return_value=__a    ) as mock_head:
                 lowerCamelCase								=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"    )
                 # This check we did call the fake head request
                 mock_head.assert_called()







     @require_tokenizers
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		mock.Mock()
           lowerCamelCase								=		5_00
           lowerCamelCase								=		{}
           lowerCamelCase								=		HTTPError
           lowerCamelCase								=		{}

           # Download this model to make sure it's in the cache.
           lowerCamelCase								=		GPTaTokenizerFast.from_pretrained("gpt2"    )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("requests.Session.request"				,			return_value=__a    ) as mock_head:
                 lowerCamelCase								=		GPTaTokenizerFast.from_pretrained("gpt2"    )
                 # This check we did call the fake head request
                 mock_head.assert_called()







     def 						_a      (self    ):






           '''simple docstring'''



           try:
                 lowerCamelCase								=		tempfile.mktemp()
                 with open(__a				,			"wb"    ) as f:
                       http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"				,			__a    )

                 lowerCamelCase								=		AlbertTokenizer.from_pretrained(__a    )
           finally:
                 os.remove(__a    )

           # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
           # the current folder and have the right name.
           if os.path.isfile("tokenizer.json"    ):
                 # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                 return
           try:
                 with open("tokenizer.json"				,			"wb"    ) as f:
                       http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"				,			__a    )
                 lowerCamelCase								=		AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"    )
                 # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                 self.assertEqual(tokenizer.vocab_size				,			10_00    )
                 # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

           finally:
                 os.remove("tokenizer.json"    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"    )





@is_staging_test
class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     _A       =			['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']







     @classmethod
     def 						_a      (cls    ):






           '''simple docstring'''



           lowerCamelCase								=		TOKEN
           HfFolder.save_token(__a    )







     @classmethod
     def 						_a      (cls    ):






           '''simple docstring'''



           try:
                 delete_repo(token=cls._token				,			repo_id="test-tokenizer"    )
           except HTTPError:
                 pass

           try:
                 delete_repo(token=cls._token				,			repo_id="valid_org/test-tokenizer-org"    )
           except HTTPError:
                 pass

           try:
                 delete_repo(token=cls._token				,			repo_id="test-dynamic-tokenizer"    )
           except HTTPError:
                 pass







     def 						_a      (self    ):






           '''simple docstring'''



           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		BertTokenizer(__a    )

           tokenizer.push_to_hub("test-tokenizer"				,			use_auth_token=self._token    )
           lowerCamelCase								=		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )

           # Reset repo
           delete_repo(token=self._token				,			repo_id="test-tokenizer"    )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__a				,			repo_id="test-tokenizer"				,			push_to_hub=__a				,			use_auth_token=self._token    )

           lowerCamelCase								=		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )







     def 						_a      (self    ):






           '''simple docstring'''



           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		BertTokenizer(__a    )

           tokenizer.push_to_hub("valid_org/test-tokenizer-org"				,			use_auth_token=self._token    )
           lowerCamelCase								=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )

           # Reset repo
           delete_repo(token=self._token				,			repo_id="valid_org/test-tokenizer-org"    )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(
                     __a				,			repo_id="valid_org/test-tokenizer-org"				,			push_to_hub=__a				,			use_auth_token=self._token    )

           lowerCamelCase								=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )







     @require_tokenizers
     def 						_a      (self    ):






           '''simple docstring'''



           CustomTokenizer.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		CustomTokenizer(__a    )

           # No fast custom tokenizer
           tokenizer.push_to_hub("test-dynamic-tokenizer"				,			use_auth_token=self._token    )

           lowerCamelCase								=		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizer"    )

           # Fast and slow custom tokenizer
           CustomTokenizerFast.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )

                 lowerCamelCase								=		BertTokenizerFast.from_pretrained(__a    )
                 bert_tokenizer.save_pretrained(__a    )
                 lowerCamelCase								=		CustomTokenizerFast.from_pretrained(__a    )

           tokenizer.push_to_hub("test-dynamic-tokenizer"				,			use_auth_token=self._token    )

           lowerCamelCase								=		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizerFast"    )
           lowerCamelCase								=		AutoTokenizer.from_pretrained(
               F"""{USER}/test-dynamic-tokenizer"""				,			use_fast=__a				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizer"    )





class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("Hello 友達"    )
           self.assertEqual(trie.data				,			{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}    )
           trie.add("Hello"    )
           trie.data
           self.assertEqual(trie.data				,			{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           self.assertEqual(trie.split("[CLS] This is a extra_id_100"    )				,			["[CLS] This is a extra_id_100"]    )
           trie.add("[CLS]"    )
           trie.add("extra_id_1"    )
           trie.add("extra_id_100"    )
           self.assertEqual(trie.split("[CLS] This is a extra_id_100"    )				,			["[CLS]", " This is a ", "extra_id_100"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("A"    )
           self.assertEqual(trie.split("ABC"    )				,			["A", "BC"]    )
           self.assertEqual(trie.split("BCA"    )				,			["BC", "A"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("TOKEN]"    )
           trie.add("[SPECIAL_TOKEN]"    )
           self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"    )				,			["This is something ", "[SPECIAL_TOKEN]"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("A"    )
           trie.add("P"    )
           trie.add("[SPECIAL_TOKEN]"    )
           self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"    )				,			["This is something ", "[SPECIAL_TOKEN]"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("AB"    )
           trie.add("B"    )
           trie.add("C"    )
           self.assertEqual(trie.split("ABC"    )				,			["AB", "C"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("ABC"    )
           trie.add("B"    )
           trie.add("CD"    )
           self.assertEqual(trie.split("ABCD"    )				,			["ABC", "D"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           lowerCamelCase								=		trie.cut_text("ABC"				,			[0, 0, 2, 1, 2, 3]    )
           self.assertEqual(__a				,			["AB", "C"]    )