






"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_       			=   '''▁'''

lowerCamelCase_       			=   {'''vocab_file''': '''spiece.model'''}

lowerCamelCase_       			=   {
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

lowerCamelCase_       			=   {
    '''google/pegasus-xsum''': 512,
}


lowerCamelCase_       			=   logging.get_logger(__name__)







class     UpperCamelCase_ (__A  ):
	__magic_name__			   =       VOCAB_FILES_NAMES

	__magic_name__			   =       VOCAB_FILES_NAMES
	__magic_name__			   =       PRETRAINED_VOCAB_FILES_MAP
	__magic_name__			   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__			   =       ['''input_ids''', '''attention_mask''']





	def __init__(   self      :				Union[str, Any]			,			lowerCAmelCase_      :				Optional[Any]			,			lowerCAmelCase_      :				List[Any]="<pad>"			,			lowerCAmelCase_      :				List[Any]="</s>"			,			lowerCAmelCase_      :				str="<unk>"			,			lowerCAmelCase_      :				List[Any]="<mask_2>"			,			lowerCAmelCase_      :				Union[str, Any]="<mask_1>"			,			lowerCAmelCase_      :				Union[str, Any]=None			,			lowerCAmelCase_      :				Optional[Any]=103			,			lowerCAmelCase_      :				Optional[Dict[str, Any]] = None			,			**lowerCAmelCase_      :				List[str]			,			)		->    None:
				UpperCAmelCase_  :				int      =      offset
				if additional_special_tokens is not None:
							if not isinstance(lowerCAmelCase_			,			lowerCAmelCase_		):
										raise TypeError(
										    f"""additional_special_tokens should be of type {type(lowerCAmelCase_		)}, but is"""
										    f""" {type(lowerCAmelCase_		)}"""		)

							UpperCAmelCase_  :				Any      =      (
							    ([mask_token_sent] + additional_special_tokens)
							    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
							    else additional_special_tokens
							)
							# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
							additional_special_tokens_extended += [
							    f"""<unk_{i}>""" for i in range(len(lowerCAmelCase_		)			,			self.offset - 1		)
							]

							if len(set(lowerCAmelCase_		)		) != len(lowerCAmelCase_		):
										raise ValueError(
										    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
										    f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""		)
							UpperCAmelCase_  :				int      =      additional_special_tokens_extended
				else:
							UpperCAmelCase_  :				Tuple      =      [mask_token_sent] if mask_token_sent is not None else []
							additional_special_tokens += [f"""<unk_{i}>""" for i in range(2			,			self.offset		)]

				UpperCAmelCase_  :				List[str]      =      {} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    eos_token=lowerCAmelCase_			,			unk_token=lowerCAmelCase_			,			mask_token=lowerCAmelCase_			,			pad_token=lowerCAmelCase_			,			mask_token_sent=lowerCAmelCase_			,			offset=lowerCAmelCase_			,			additional_special_tokens=lowerCAmelCase_			,			sp_model_kwargs=self.sp_model_kwargs			,			**lowerCAmelCase_			,			)
				UpperCAmelCase_  :				Dict      =      mask_token_sent
				UpperCAmelCase_  :				int      =      vocab_file
				UpperCAmelCase_  :				int      =      spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(lowerCAmelCase_		)

				# add special tokens to encoder dict
				UpperCAmelCase_  :				Dict[int, str]      =      {
				    0: self.pad_token,
				    1: self.eos_token,
				}

				if self.mask_token_sent is not None:
							self.encoder.update(
							    {
							        2: self.mask_token_sent,
							        3: self.mask_token,
							    }		)

				if self.offset > 0:
							# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
							# mask_token_sent is already added to list -> so start at 1
							self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1			,			self.offset - 1		)}		)

				UpperCAmelCase_  :				Dict[str, int]      =      {v: k for k, v in self.encoder.items()}





	@property
	def   _SCREAMING_SNAKE_CASE   (   self      :				Any		)		->    int:
				return len(self.sp_model		) + self.offset





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]		)		->    Dict[str, int]:
				UpperCAmelCase_  :				Any      =      {self.convert_ids_to_tokens(lowerCAmelCase_		): i for i in range(self.vocab_size		)}
				vocab.update(self.added_tokens_encoder		)
				return vocab





	def __getstate__(   self      :				Tuple		)		->    str:
				UpperCAmelCase_  :				int      =      self.__dict__.copy()
				UpperCAmelCase_  :				Optional[int]      =      None
				return state





	def __setstate__(   self      :				Union[str, Any]			,			lowerCAmelCase_      :				Dict		)		->    str:
				UpperCAmelCase_  :				List[str]      =      d

				# for backward compatibility
				if not hasattr(self			,			"sp_model_kwargs"		):
							UpperCAmelCase_  :				Any      =      {}

				UpperCAmelCase_  :				Tuple      =      spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(self.vocab_file		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				str			,			lowerCAmelCase_      :				str		)		->    List[str]:
				return self.sp_model.encode(lowerCAmelCase_			,			out_type=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[int]			,			lowerCAmelCase_      :				str		)		->    int:
				if token in self.decoder:
							return self.decoder[token]
				elif token in self.added_tokens_decoder:
							return self.added_tokens_decoder[token]
				UpperCAmelCase_  :				List[str]      =      self.sp_model.piece_to_id(lowerCAmelCase_		)
				return sp_id + self.offset





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				int		)		->    str:
				if index in self.encoder:
							return self.encoder[index]
				elif index in self.added_tokens_encoder:
							return self.added_tokens_encoder[index]
				else:
							UpperCAmelCase_  :				Optional[int]      =      self.sp_model.IdToPiece(index - self.offset		)
				return token





	def   _SCREAMING_SNAKE_CASE   (   self      :				str			,			lowerCAmelCase_      :				Any		)		->    Any:
				UpperCAmelCase_  :				Union[str, Any]      =      []
				UpperCAmelCase_  :				Any      =      ""
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										out_string += self.sp_model.decode(lowerCAmelCase_		) + token
										UpperCAmelCase_  :				int      =      []
							else:
										current_sub_tokens.append(lowerCAmelCase_		)
				out_string += self.sp_model.decode(lowerCAmelCase_		)
				return out_string.strip()





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any			,			lowerCAmelCase_      :				Any=False		)		->    List[Any]:
				return 1





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]			,			lowerCAmelCase_      :				Union[str, Any]		)		->    Dict:
				UpperCAmelCase_  :				int      =      set(self.all_special_ids		)  # call it once instead of inside list comp
				all_special_ids.remove(self.unk_token_id		)  # <unk> is only sometimes special

				return [1 if x in all_special_ids else 0 for x in seq]





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				List			,			lowerCAmelCase_      :				Optional[List] = None			,			lowerCAmelCase_      :				bool = False		)		->    List[int]:
				if already_has_special_tokens:
							return self._special_token_mask(lowerCAmelCase_		)
				elif token_ids_a is None:
							return self._special_token_mask(lowerCAmelCase_		) + [1]
				else:
							return self._special_token_mask(token_ids_a + token_ids_a		) + [1]





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[Any]			,			lowerCAmelCase_      :				Union[str, Any]			,			lowerCAmelCase_      :				Union[str, Any]=None		)		->    List[int]:
				if token_ids_a is None:
							return token_ids_a + [self.eos_token_id]
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return token_ids_a + token_ids_a + [self.eos_token_id]





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[Any]			,			lowerCAmelCase_      :				str			,			lowerCAmelCase_      :				Optional[str] = None		)		->    Tuple[str]:
				if not os.path.isdir(lowerCAmelCase_		):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
							return
				UpperCAmelCase_  :				Dict      =      os.path.join(
				    lowerCAmelCase_			,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCAmelCase_		) and os.path.isfile(self.vocab_file		):
							copyfile(self.vocab_file			,			lowerCAmelCase_		)
				elif not os.path.isfile(self.vocab_file		):
							with open(lowerCAmelCase_			,			"wb"		) as fi:
										UpperCAmelCase_  :				int      =      self.sp_model.serialized_model_proto()
										fi.write(lowerCAmelCase_		)

				return (out_vocab_file,)











"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase_       			=   {
    '''configuration_funnel''': ['''FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FunnelConfig'''],
    '''convert_funnel_original_tf_checkpoint_to_pytorch''': [],
    '''tokenization_funnel''': ['''FunnelTokenizer'''],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   ['''FunnelTokenizerFast''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   [
			    '''FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''FunnelBaseModel''',
			    '''FunnelForMaskedLM''',
			    '''FunnelForMultipleChoice''',
			    '''FunnelForPreTraining''',
			    '''FunnelForQuestionAnswering''',
			    '''FunnelForSequenceClassification''',
			    '''FunnelForTokenClassification''',
			    '''FunnelModel''',
			    '''FunnelPreTrainedModel''',
			    '''load_tf_weights_in_funnel''',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   [
			    '''TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''TFFunnelBaseModel''',
			    '''TFFunnelForMaskedLM''',
			    '''TFFunnelForMultipleChoice''',
			    '''TFFunnelForPreTraining''',
			    '''TFFunnelForQuestionAnswering''',
			    '''TFFunnelForSequenceClassification''',
			    '''TFFunnelForTokenClassification''',
			    '''TFFunnelModel''',
			    '''TFFunnelPreTrainedModel''',
			]


if TYPE_CHECKING:
			from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
			from .tokenization_funnel import FunnelTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_funnel_fast import FunnelTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_funnel import (
						    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    FunnelBaseModel,
						    FunnelForMaskedLM,
						    FunnelForMultipleChoice,
						    FunnelForPreTraining,
						    FunnelForQuestionAnswering,
						    FunnelForSequenceClassification,
						    FunnelForTokenClassification,
						    FunnelModel,
						    FunnelPreTrainedModel,
						    load_tf_weights_in_funnel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_funnel import (
						    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFFunnelBaseModel,
						    TFFunnelForMaskedLM,
						    TFFunnelForMultipleChoice,
						    TFFunnelForPreTraining,
						    TFFunnelForQuestionAnswering,
						    TFFunnelForSequenceClassification,
						    TFFunnelForTokenClassification,
						    TFFunnelModel,
						    TFFunnelPreTrainedModel,
						)

else:
			import sys

			lowerCamelCase_       			=   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



