




"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


A   =       logging.get_logger(__name__)


A   =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
A   =       {
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json""",
    },
}

A   =       {
    """RUCAIBox/mvp""": 1_024,
}



class      a__				(	__magic_name__   ):
			lowercase_    =	VOCAB_FILES_NAMES
			lowercase_    =	PRETRAINED_VOCAB_FILES_MAP
			lowercase_    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase_    =	["input_ids", "attention_mask"]
			lowercase_    =	MvpTokenizer


			def __init__(      self      :    str      ,  UpperCamelCase_      :    str=None      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    str="replace"      ,  UpperCamelCase_      :    List[Any]="<s>"      ,  UpperCamelCase_      :    Any="</s>"      ,  UpperCamelCase_      :    Dict="</s>"      ,  UpperCamelCase_      :    Union[str, Any]="<s>"      ,  UpperCamelCase_      :    Union[str, Any]="<unk>"      ,  UpperCamelCase_      :    Optional[Any]="<pad>"      ,  UpperCamelCase_      :    Optional[Any]="<mask>"      ,  UpperCamelCase_      :    int=False      ,  UpperCamelCase_      :    Any=True      ,  **UpperCamelCase_      :    Optional[int]      ,  ):


						"""simple docstring"""




						super().__init__(
						    UpperCamelCase_      ,  UpperCamelCase_      ,  tokenizer_file=UpperCamelCase_      ,  errors=UpperCamelCase_      ,  bos_token=UpperCamelCase_      ,  eos_token=UpperCamelCase_      ,  sep_token=UpperCamelCase_      ,  cls_token=UpperCamelCase_      ,  unk_token=UpperCamelCase_      ,  pad_token=UpperCamelCase_      ,  mask_token=UpperCamelCase_      ,  add_prefix_space=UpperCamelCase_      ,  trim_offsets=UpperCamelCase_      ,  **UpperCamelCase_      ,  )

						__UpperCAmelCase	: Tuple		=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
						if pre_tok_state.get("add_prefix_space"      ,  UpperCamelCase_) != add_prefix_space:
									__UpperCAmelCase	: str		=					getattr(UpperCamelCase_      ,  pre_tok_state.pop("type"))
									__UpperCAmelCase	: Optional[Any]		=					add_prefix_space
									__UpperCAmelCase	: Optional[int]		=					pre_tok_class(**UpperCamelCase_)

						__UpperCAmelCase	: str		=					add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						__UpperCAmelCase	: List[str]		=					"post_processor"
						__UpperCAmelCase	: Dict		=					getattr(self.backend_tokenizer      ,  UpperCamelCase_      ,  UpperCamelCase_)
						if tokenizer_component_instance:
									__UpperCAmelCase	: Dict		=					json.loads(tokenizer_component_instance.__getstate__())

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
												__UpperCAmelCase	: Optional[Any]		=					tuple(state["sep"])
									if "cls" in state:
												__UpperCAmelCase	: Tuple		=					tuple(state["cls"])

									__UpperCAmelCase	: List[str]		=					False

									if state.get("add_prefix_space"      ,  UpperCamelCase_) != add_prefix_space:
												__UpperCAmelCase	: Any		=					add_prefix_space
												__UpperCAmelCase	: Tuple		=					True

									if state.get("trim_offsets"      ,  UpperCamelCase_) != trim_offsets:
												__UpperCAmelCase	: Optional[int]		=					trim_offsets
												__UpperCAmelCase	: int		=					True

									if changes_to_apply:
												__UpperCAmelCase	: Optional[Any]		=					getattr(UpperCamelCase_      ,  state.pop("type"))
												__UpperCAmelCase	: Union[str, Any]		=					component_class(**UpperCamelCase_)
												setattr(self.backend_tokenizer      ,  UpperCamelCase_      ,  UpperCamelCase_)


			@property
			def       a_     (      self      :    Optional[int]):


						"""simple docstring"""




						if self._mask_token is None:
									if self.verbose:
												logger.error("Using mask_token, but it is not set yet.")
									return None
						return str(self._mask_token)


			@mask_token.setter
			def       a_     (      self      :    List[str]      ,  UpperCamelCase_      :    Dict):


						"""simple docstring"""




						__UpperCAmelCase	: Dict		=					AddedToken(UpperCamelCase_      ,  lstrip=UpperCamelCase_      ,  rstrip=UpperCamelCase_) if isinstance(UpperCamelCase_      ,  UpperCamelCase_) else value
						__UpperCAmelCase	: Dict		=					value


			def       a_     (      self      :    str      ,  *UpperCamelCase_      :    List[str]      ,  **UpperCamelCase_      :    Optional[int]):


						"""simple docstring"""




						__UpperCAmelCase	: Tuple		=					kwargs.get("is_split_into_words"      ,  UpperCamelCase_)

						if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    "to use it with pretokenized inputs.")

						return super()._batch_encode_plus(*UpperCamelCase_      ,  **UpperCamelCase_)


			def       a_     (      self      :    List[Any]      ,  *UpperCamelCase_      :    Dict      ,  **UpperCamelCase_      :    Tuple):


						"""simple docstring"""




						__UpperCAmelCase	: Union[str, Any]		=					kwargs.get("is_split_into_words"      ,  UpperCamelCase_)

						if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    "to use it with pretokenized inputs.")

						return super()._encode_plus(*UpperCamelCase_      ,  **UpperCamelCase_)


			def       a_     (      self      :    Union[str, Any]      ,  UpperCamelCase_      :    str      ,  UpperCamelCase_      :    Optional[str] = None):


						"""simple docstring"""




						__UpperCAmelCase	: Union[str, Any]		=					self._tokenizer.model.save(UpperCamelCase_      ,  name=UpperCamelCase_)
						return tuple(UpperCamelCase_)


			def       a_     (      self      :    Tuple      ,  UpperCamelCase_      :    str      ,  UpperCamelCase_      :    Optional[int]=None):


						"""simple docstring"""




						__UpperCAmelCase	: str		=					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
									return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


			def       a_     (      self      :    List[str]      ,  UpperCamelCase_      :    List[int]      ,  UpperCamelCase_      :    Optional[List[int]] = None):


						"""simple docstring"""




						__UpperCAmelCase	: List[Any]		=					[self.sep_token_id]
						__UpperCAmelCase	: Union[str, Any]		=					[self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]








"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


A   =       logging.get_logger(__name__)

A   =       {
    """bigcode/gpt_bigcode-santacoder""": """https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json""",
}



class      a__				(	__magic_name__   ):
			lowercase_    =	"gpt_bigcode"
			lowercase_    =	["past_key_values"]
			lowercase_    =	{
			    "hidden_size": "n_embd",
			    "max_position_embeddings": "n_positions",
			    "num_attention_heads": "n_head",
			    "num_hidden_layers": "n_layer",
			}


			def __init__(      self      :    Any      ,  UpperCamelCase_      :    Tuple=50257      ,  UpperCamelCase_      :    Dict=1024      ,  UpperCamelCase_      :    Optional[Any]=768      ,  UpperCamelCase_      :    Optional[Any]=12      ,  UpperCamelCase_      :    Any=12      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    List[Any]="gelu_pytorch_tanh"      ,  UpperCamelCase_      :    Optional[Any]=0.1      ,  UpperCamelCase_      :    Union[str, Any]=0.1      ,  UpperCamelCase_      :    Tuple=0.1      ,  UpperCamelCase_      :    Tuple=1e-5      ,  UpperCamelCase_      :    List[str]=0.02      ,  UpperCamelCase_      :    Dict=True      ,  UpperCamelCase_      :    Optional[int]=True      ,  UpperCamelCase_      :    str=50256      ,  UpperCamelCase_      :    Union[str, Any]=50256      ,  UpperCamelCase_      :    Dict=True      ,  UpperCamelCase_      :    List[str]=True      ,  UpperCamelCase_      :    Union[str, Any]=True      ,  **UpperCamelCase_      :    Union[str, Any]      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: Tuple		=					vocab_size
						__UpperCAmelCase	: Optional[int]		=					n_positions
						__UpperCAmelCase	: Tuple		=					n_embd
						__UpperCAmelCase	: str		=					n_layer
						__UpperCAmelCase	: Dict		=					n_head
						__UpperCAmelCase	: Optional[Any]		=					n_inner
						__UpperCAmelCase	: Optional[Any]		=					activation_function
						__UpperCAmelCase	: List[str]		=					resid_pdrop
						__UpperCAmelCase	: List[Any]		=					embd_pdrop
						__UpperCAmelCase	: Optional[Any]		=					attn_pdrop
						__UpperCAmelCase	: Dict		=					layer_norm_epsilon
						__UpperCAmelCase	: List[str]		=					initializer_range
						__UpperCAmelCase	: int		=					scale_attn_weights
						__UpperCAmelCase	: Tuple		=					use_cache
						__UpperCAmelCase	: List[Any]		=					attention_softmax_in_fpaa
						__UpperCAmelCase	: Any		=					scale_attention_softmax_in_fpaa
						__UpperCAmelCase	: str		=					multi_query

						__UpperCAmelCase	: int		=					bos_token_id
						__UpperCAmelCase	: str		=					eos_token_id

						super().__init__(bos_token_id=UpperCamelCase_      ,  eos_token_id=UpperCamelCase_      ,  **UpperCamelCase_)


