




"""simple docstring"""





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class      a__				(	__magic_name__							,    unittest.TestCase   ):
			lowercase_    =	DebertaTokenizer
			lowercase_    =	True
			lowercase_    =	DebertaTokenizerFast


			def       a_     (      self      :    Tuple):


						"""simple docstring"""




						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						__UpperCAmelCase	: Any		=					[
						    "l",
						    "o",
						    "w",
						    "e",
						    "r",
						    "s",
						    "t",
						    "i",
						    "d",
						    "n",
						    "\u0120",
						    "\u0120l",
						    "\u0120n",
						    "\u0120lo",
						    "\u0120low",
						    "er",
						    "\u0120lowest",
						    "\u0120newer",
						    "\u0120wider",
						    "[UNK]",
						]
						__UpperCAmelCase	: Optional[int]		=					dict(zip(UpperCamelCase_      ,  range(len(UpperCamelCase_))))
						__UpperCAmelCase	: Union[str, Any]		=					["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
						__UpperCAmelCase	: Optional[Any]		=					{"unk_token": "[UNK]"}

						__UpperCAmelCase	: List[str]		=					os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES["vocab_file"])
						__UpperCAmelCase	: List[str]		=					os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES["merges_file"])
						with open(self.vocab_file      ,  "w"      ,  encoding="utf-8") as fp:
									fp.write(json.dumps(UpperCamelCase_) + "\n")
						with open(self.merges_file      ,  "w"      ,  encoding="utf-8") as fp:
									fp.write("\n".join(UpperCamelCase_))


			def       a_     (      self      :    Tuple      ,  **UpperCamelCase_      :    int):


						"""simple docstring"""




						kwargs.update(self.special_tokens_map)
						return self.tokenizer_class.from_pretrained(self.tmpdirname      ,  **UpperCamelCase_)


			def       a_     (      self      :    Dict      ,  UpperCamelCase_      :    List[str]):


						"""simple docstring"""




						__UpperCAmelCase	: int		=					"lower newer"
						__UpperCAmelCase	: Dict		=					"lower newer"
						return input_text, output_text


			def       a_     (      self      :    int):


						"""simple docstring"""




						__UpperCAmelCase	: List[str]		=					self.get_tokenizer()
						__UpperCAmelCase	: Tuple		=					"lower newer"
						__UpperCAmelCase	: int		=					["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
						__UpperCAmelCase	: List[Any]		=					tokenizer.tokenize(UpperCamelCase_)
						self.assertListEqual(UpperCamelCase_      ,  UpperCamelCase_)

						__UpperCAmelCase	: int		=					tokens + [tokenizer.unk_token]
						__UpperCAmelCase	: List[str]		=					[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_)      ,  UpperCamelCase_)


			def       a_     (      self      :    int):


						"""simple docstring"""




						__UpperCAmelCase	: Tuple		=					self.get_tokenizer()
						__UpperCAmelCase	: Tuple		=					tokenizer("Hello"      ,  "World")
						__UpperCAmelCase	: List[Any]		=					[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
						self.assertListEqual(tokd["token_type_ids"]      ,  UpperCamelCase_)


			@slow
			def       a_     (      self      :    Optional[Any]):


						"""simple docstring"""




						__UpperCAmelCase	: Any		=					self.tokenizer_class.from_pretrained("microsoft/deberta-base")

						__UpperCAmelCase	: Union[str, Any]		=					tokenizer.encode("sequence builders"      ,  add_special_tokens=UpperCamelCase_)
						__UpperCAmelCase	: str		=					tokenizer.encode("multi-sequence build"      ,  add_special_tokens=UpperCamelCase_)

						__UpperCAmelCase	: Optional[int]		=					tokenizer.encode(
						    "sequence builders"      ,  add_special_tokens=UpperCamelCase_      ,  add_prefix_space=UpperCamelCase_)
						__UpperCAmelCase	: Tuple		=					tokenizer.encode(
						    "sequence builders"      ,  "multi-sequence build"      ,  add_special_tokens=UpperCamelCase_      ,  add_prefix_space=UpperCamelCase_)

						__UpperCAmelCase	: List[Any]		=					tokenizer.build_inputs_with_special_tokens(UpperCamelCase_)
						__UpperCAmelCase	: Optional[int]		=					tokenizer.build_inputs_with_special_tokens(UpperCamelCase_      ,  UpperCamelCase_)

						assert encoded_sentence == encoded_text_from_decode
						assert encoded_pair == encoded_pair_from_decode


			@slow
			def       a_     (      self      :    Dict):


						"""simple docstring"""




						__UpperCAmelCase	: List[str]		=					[self.tokenizer_class]
						if self.test_rust_tokenizer:
									tokenizer_classes.append(self.rust_tokenizer_class)

						for tokenizer_class in tokenizer_classes:
									__UpperCAmelCase	: Dict		=					tokenizer_class.from_pretrained("microsoft/deberta-base")

									__UpperCAmelCase	: Any		=					[
									    "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
									    "ALBERT incorporates two parameter reduction techniques",
									    "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
									    " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
									    " vocabulary embedding.",
									]

									__UpperCAmelCase	: List[Any]		=					tokenizer(UpperCamelCase_      ,  padding=UpperCamelCase_)
									__UpperCAmelCase	: int		=					[tokenizer.decode(UpperCamelCase_      ,  skip_special_tokens=UpperCamelCase_) for seq in encoding["input_ids"]]

									# fmt: off
									__UpperCAmelCase	: Optional[Any]		=					{
									    "input_ids": [
									        [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
									    ],
									    "token_type_ids": [
									        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
									    ],
									    "attention_mask": [
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
									    ]
									}
									# fmt: on

									__UpperCAmelCase	: Any		=					[
									    "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
									    "ALBERT incorporates two parameter reduction techniques",
									    "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
									    " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
									    " vocabulary embedding.",
									]

									self.assertDictEqual(encoding.data      ,  UpperCamelCase_)

									for expected, decoded in zip(UpperCamelCase_      ,  UpperCamelCase_):
												self.assertEqual(UpperCamelCase_      ,  UpperCamelCase_)








"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


A   =       logging.get_logger(__name__)

A   =       {
    """bigcode/gpt_bigcode-santacoder""": """https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json""",
}



class      a__				(	__magic_name__   ):
			lowercase_    =	"gpt_bigcode"
			lowercase_    =	["past_key_values"]
			lowercase_    =	{
			    "hidden_size": "n_embd",
			    "max_position_embeddings": "n_positions",
			    "num_attention_heads": "n_head",
			    "num_hidden_layers": "n_layer",
			}


			def __init__(      self      :    Any      ,  UpperCamelCase_      :    Tuple=50257      ,  UpperCamelCase_      :    Dict=1024      ,  UpperCamelCase_      :    Optional[Any]=768      ,  UpperCamelCase_      :    Optional[Any]=12      ,  UpperCamelCase_      :    Any=12      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    List[Any]="gelu_pytorch_tanh"      ,  UpperCamelCase_      :    Optional[Any]=0.1      ,  UpperCamelCase_      :    Union[str, Any]=0.1      ,  UpperCamelCase_      :    Tuple=0.1      ,  UpperCamelCase_      :    Tuple=1e-5      ,  UpperCamelCase_      :    List[str]=0.02      ,  UpperCamelCase_      :    Dict=True      ,  UpperCamelCase_      :    Optional[int]=True      ,  UpperCamelCase_      :    str=50256      ,  UpperCamelCase_      :    Union[str, Any]=50256      ,  UpperCamelCase_      :    Dict=True      ,  UpperCamelCase_      :    List[str]=True      ,  UpperCamelCase_      :    Union[str, Any]=True      ,  **UpperCamelCase_      :    Union[str, Any]      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: Tuple		=					vocab_size
						__UpperCAmelCase	: Optional[int]		=					n_positions
						__UpperCAmelCase	: Tuple		=					n_embd
						__UpperCAmelCase	: str		=					n_layer
						__UpperCAmelCase	: Dict		=					n_head
						__UpperCAmelCase	: Optional[Any]		=					n_inner
						__UpperCAmelCase	: Optional[Any]		=					activation_function
						__UpperCAmelCase	: List[str]		=					resid_pdrop
						__UpperCAmelCase	: List[Any]		=					embd_pdrop
						__UpperCAmelCase	: Optional[Any]		=					attn_pdrop
						__UpperCAmelCase	: Dict		=					layer_norm_epsilon
						__UpperCAmelCase	: List[str]		=					initializer_range
						__UpperCAmelCase	: int		=					scale_attn_weights
						__UpperCAmelCase	: Tuple		=					use_cache
						__UpperCAmelCase	: List[Any]		=					attention_softmax_in_fpaa
						__UpperCAmelCase	: Any		=					scale_attention_softmax_in_fpaa
						__UpperCAmelCase	: str		=					multi_query

						__UpperCAmelCase	: int		=					bos_token_id
						__UpperCAmelCase	: str		=					eos_token_id

						super().__init__(bos_token_id=UpperCamelCase_      ,  eos_token_id=UpperCamelCase_      ,  **UpperCamelCase_)


