'''simple docstring'''

class 				_lowercase	:
						'''simple docstring'''


						def __init__(				self							)			->			str:



									'''simple docstring'''



									UpperCAmelCase__							:				Optional[Any]   				=  0
									UpperCAmelCase__							:				Tuple   				=  0
									UpperCAmelCase__							:				str   				=  {}




						def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			Dict:



									'''simple docstring'''



									if vertex not in self.adjacency:
												UpperCAmelCase__							:				Any   				=  {}
												self.num_vertices += 1




						def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_					,lowerCamelCase_							)			->			Tuple:



									'''simple docstring'''



									self.add_vertex(lowerCamelCase_							)
									self.add_vertex(lowerCamelCase_							)

									if head == tail:
												return

									UpperCAmelCase__							:				str   				=  weight
									UpperCAmelCase__							:				Union[str, Any]   				=  weight




						def 		lowerCAmelCase__					(				self							)			->			int:



									'''simple docstring'''



									UpperCAmelCase__							:				Optional[int]   				=  self.get_edges()
									for edge in edges:
												UpperCAmelCase__				,      UpperCAmelCase__				,      UpperCAmelCase__							:				List[Any]   				=  edge
												edges.remove((tail, head, weight)							)
									for i in range(len(lowerCamelCase_							)							):
												UpperCAmelCase__							:				int   				=  list(edges[i]							)

									edges.sort(key=lambda lowerCamelCase_							: e[2]							)
									for i in range(len(lowerCamelCase_							) - 1							):
												if edges[i][2] >= edges[i + 1][2]:
															UpperCAmelCase__							:				Union[str, Any]   				=  edges[i][2] + 1
									for edge in edges:
												UpperCAmelCase__				,      UpperCAmelCase__				,      UpperCAmelCase__							:				Dict   				=  edge
												UpperCAmelCase__							:				int   				=  weight
												UpperCAmelCase__							:				str   				=  weight




						def __str__(				self							)			->			int:



									'''simple docstring'''



									UpperCAmelCase__							:				List[str]   				=  ''''''
									for tail in self.adjacency:
												for head in self.adjacency[tail]:
															UpperCAmelCase__							:				Union[str, Any]   				=  self.adjacency[head][tail]
															string += f'''{head} -> {tail} == {weight}\n'''
									return string.rstrip('''\n'''							)




						def 		lowerCAmelCase__					(				self							)			->			Optional[int]:



									'''simple docstring'''



									UpperCAmelCase__							:				Optional[int]   				=  []
									for tail in self.adjacency:
												for head in self.adjacency[tail]:
															output.append((tail, head, self.adjacency[head][tail])							)
									return output




						def 		lowerCAmelCase__					(				self							)			->			str:



									'''simple docstring'''



									return self.adjacency.keys()




						@staticmethod
						def 		lowerCAmelCase__					(				lowerCamelCase_=None					,lowerCamelCase_=None							)			->			int:



									'''simple docstring'''



									UpperCAmelCase__							:				List[Any]   				=  Graph()
									if vertices is None:
												UpperCAmelCase__							:				Dict   				=  []
									if edges is None:
												UpperCAmelCase__							:				Optional[int]   				=  []
									for vertex in vertices:
												g.add_vertex(lowerCamelCase_							)
									for edge in edges:
												g.add_edge(*lowerCamelCase_							)
									return g


						class 				_lowercase	:
												'''simple docstring'''


												def __init__(				self							)			->			str:



															'''simple docstring'''



															UpperCAmelCase__							:				Optional[int]   				=  {}
															UpperCAmelCase__							:				List[Any]   				=  {}




												def __len__(				self							)			->			Union[str, Any]:



															'''simple docstring'''



															return len(self.parent							)




												def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			str:



															'''simple docstring'''



															if item in self.parent:
																		return self.find(lowerCamelCase_							)

															UpperCAmelCase__							:				Optional[int]   				=  item
															UpperCAmelCase__							:				List[Any]   				=  0
															return item




												def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			Union[str, Any]:



															'''simple docstring'''



															if item not in self.parent:
																		return self.make_set(lowerCamelCase_							)
															if item != self.parent[item]:
																		UpperCAmelCase__							:				Tuple   				=  self.find(self.parent[item]							)
															return self.parent[item]




												def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_							)			->			List[str]:



															'''simple docstring'''



															UpperCAmelCase__							:				Tuple   				=  self.find(lowerCamelCase_							)
															UpperCAmelCase__							:				str   				=  self.find(lowerCamelCase_							)

															if roota == roota:
																		return roota

															if self.rank[roota] > self.rank[roota]:
																		UpperCAmelCase__							:				Any   				=  roota
																		return roota

															if self.rank[roota] < self.rank[roota]:
																		UpperCAmelCase__							:				Union[str, Any]   				=  roota
																		return roota

															if self.rank[roota] == self.rank[roota]:
																		self.rank[roota] += 1
																		UpperCAmelCase__							:				Dict   				=  roota
																		return roota
															return None




						@staticmethod
						def 		lowerCAmelCase__					(				lowerCamelCase_							)			->			Union[str, Any]:



									'''simple docstring'''



									UpperCAmelCase__							:				Tuple   				=  graph.num_vertices

									UpperCAmelCase__							:				Any   				=  Graph.UnionFind()
									UpperCAmelCase__							:				int   				=  []
									while num_components > 1:
												UpperCAmelCase__							:				Optional[Any]   				=  {}
												for vertex in graph.get_vertices():
															UpperCAmelCase__							:				Optional[int]   				=  -1

												UpperCAmelCase__							:				List[Any]   				=  graph.get_edges()
												for edge in edges:
															UpperCAmelCase__				,      UpperCAmelCase__				,      UpperCAmelCase__							:				Optional[Any]   				=  edge
															edges.remove((tail, head, weight)							)
												for edge in edges:
															UpperCAmelCase__				,      UpperCAmelCase__				,      UpperCAmelCase__							:				str   				=  edge
															UpperCAmelCase__							:				Optional[Any]   				=  union_find.find(lowerCamelCase_							)
															UpperCAmelCase__							:				Tuple   				=  union_find.find(lowerCamelCase_							)
															if seta != seta:
																		if cheap_edge[seta] == -1 or cheap_edge[seta][2] > weight:
																					UpperCAmelCase__							:				Dict   				=  [head, tail, weight]

																		if cheap_edge[seta] == -1 or cheap_edge[seta][2] > weight:
																					UpperCAmelCase__							:				int   				=  [head, tail, weight]
												for vertex in cheap_edge:
															if cheap_edge[vertex] != -1:
																		UpperCAmelCase__				,      UpperCAmelCase__				,      UpperCAmelCase__							:				Optional[int]   				=  cheap_edge[vertex]
																		if union_find.find(lowerCamelCase_							) != union_find.find(lowerCamelCase_							):
																					union_find.union(lowerCamelCase_					,lowerCamelCase_							)
																					mst_edges.append(cheap_edge[vertex]							)
																					UpperCAmelCase__							:				Optional[Any]   				=  num_components - 1
									UpperCAmelCase__							:				List[Any]   				=  Graph.build(edges=lowerCamelCase_							)
									return mst





'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__     :	Union[str, Any]				=			logging.get_logger(__name__)

UpperCamelCase__     :	str				=			{'vocab_file': 'sentencepiece.model'}

UpperCamelCase__     :	int				=			{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
}

UpperCamelCase__     :	Any				=			{
    'google/rembert': 256,
}


class 				_lowercase	(      lowerCAmelCase   ):
      '''simple docstring'''


      UpperCAmelCase_      :      List[Any]      							=   VOCAB_FILES_NAMES
      UpperCAmelCase_      :      List[Any]      							=   PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_      :      Tuple      							=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




      def __init__(				self					,lowerCamelCase_					,lowerCamelCase_=False					,lowerCamelCase_=True					,lowerCamelCase_=True					,lowerCamelCase_="[CLS]"					,lowerCamelCase_="[SEP]"					,lowerCamelCase_="[UNK]"					,lowerCamelCase_="[SEP]"					,lowerCamelCase_="[PAD]"					,lowerCamelCase_="[CLS]"					,lowerCamelCase_="[MASK]"					,**lowerCamelCase_					,)			->			Tuple:



         '''simple docstring'''



         super().__init__(
             do_lower_case=lowerCamelCase_					,remove_space=lowerCamelCase_					,keep_accents=lowerCamelCase_					,bos_token=lowerCamelCase_					,eos_token=lowerCamelCase_					,unk_token=lowerCamelCase_					,sep_token=lowerCamelCase_					,pad_token=lowerCamelCase_					,cls_token=lowerCamelCase_					,mask_token=lowerCamelCase_					,**lowerCamelCase_					,)

         UpperCAmelCase__							:				Union[str, Any]   				=  do_lower_case
         UpperCAmelCase__							:				str   				=  remove_space
         UpperCAmelCase__							:				Optional[Any]   				=  keep_accents
         UpperCAmelCase__							:				List[Any]   				=  vocab_file

         UpperCAmelCase__							:				Dict   				=  spm.SentencePieceProcessor()
         self.sp_model.Load(lowerCamelCase_							)




      @property
      def 		lowerCAmelCase__					(				self							)			->			Optional[Any]:



         '''simple docstring'''



         return len(self.sp_model							)




      def 		lowerCAmelCase__					(				self							)			->			Dict:



         '''simple docstring'''



         UpperCAmelCase__							:				Optional[int]   				=  {self.convert_ids_to_tokens(lowerCamelCase_							): i for i in range(self.vocab_size							)}
         vocab.update(self.added_tokens_encoder							)
         return vocab




      def __getstate__(				self							)			->			Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				Optional[int]   				=  self.__dict__.copy()
         UpperCAmelCase__							:				Any   				=  None
         return state




      def __setstate__(				self					,lowerCamelCase_							)			->			Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				List[str]   				=  d
         UpperCAmelCase__							:				Any   				=  spm.SentencePieceProcessor()
         self.sp_model.Load(self.vocab_file							)




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_=False							)			->			Tuple:



         '''simple docstring'''



         UpperCAmelCase__							:				str   				=  self.sp_model.EncodeAsPieces(lowerCamelCase_							)
         return pieces




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			str:



         '''simple docstring'''



         return self.sp_model.PieceToId(lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			Optional[int]:



         '''simple docstring'''



         return self.sp_model.IdToPiece(lowerCamelCase_							)




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_							)			->			List[Any]:



         '''simple docstring'''



         UpperCAmelCase__							:				List[Any]   				=  self.sp_model.decode_pieces(lowerCamelCase_							)
         return out_string




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_ = None							)			->			List[int]:



         '''simple docstring'''



         UpperCAmelCase__							:				Union[str, Any]   				=  [self.sep_token_id]
         UpperCAmelCase__							:				List[Any]   				=  [self.cls_token_id]
         if token_ids_a is None:
            return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_ = None					,lowerCamelCase_ = False							)			->			List[int]:



         '''simple docstring'''



         if already_has_special_tokens:
            if token_ids_a is not None:
               raise ValueError(
                   '''You should not supply a second sequence if the provided sequence of '''
                   '''ids is already formatted with special tokens for the model.'''							)
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
            return [1] + ([0] * len(lowerCamelCase_							)) + [1] + ([0] * len(lowerCamelCase_							)) + [1]
         return [1] + ([0] * len(lowerCamelCase_							)) + [1]




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_ = None							)			->			List[int]:



         '''simple docstring'''



         UpperCAmelCase__							:				Tuple   				=  [self.sep_token_id]
         UpperCAmelCase__							:				List[Any]   				=  [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]




      def 		lowerCAmelCase__					(				self					,lowerCamelCase_					,lowerCamelCase_ = None							)			->			Tuple[str]:



         '''simple docstring'''



         if not os.path.isdir(lowerCamelCase_							):
            logger.error('''Vocabulary path ({}) should be a directory'''.format(lowerCamelCase_							)							)
            return
         UpperCAmelCase__							:				Union[str, Any]   				=  os.path.join(
             lowerCamelCase_					,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCamelCase_							):
            copyfile(self.vocab_file					,lowerCamelCase_							)

         return (out_vocab_file,)




