
"""simple docstring"""





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


UpperCamelCase_					:						Optional[Any]				= logging.get_logger(__name__)


UpperCamelCase_					:						Union[str, Any]				= {
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				"codegen"
			snake_case	      =				{
			    "max_position_embeddings": "n_positions",
			    "hidden_size": "n_embd",
			    "num_attention_heads": "n_head",
			    "num_hidden_layers": "n_layer",
			}




			def __init__(				self						:					int       ,   _snake_case						:					int=50_400       ,   _snake_case						:					Any=2_048       ,   _snake_case						:					List[str]=2_048       ,   _snake_case						:					Optional[Any]=4_096       ,   _snake_case						:					str=28       ,   _snake_case						:					Dict=16       ,   _snake_case						:					Tuple=64       ,   _snake_case						:					Union[str, Any]=None       ,   _snake_case						:					int="gelu_new"       ,   _snake_case						:					int=0.0       ,   _snake_case						:					Dict=0.0       ,   _snake_case						:					Optional[Any]=0.0       ,   _snake_case						:					Tuple=1e-5       ,   _snake_case						:					Tuple=0.0_2       ,   _snake_case						:					Tuple=True       ,   _snake_case						:					Optional[Any]=50_256       ,   _snake_case						:					Union[str, Any]=50_256       ,   _snake_case						:					str=False       ,   **_snake_case						:					Tuple       ,   )      -> int:







								"""simple docstring"""
								A_				 =						vocab_size
								A_				 =						n_ctx
								A_				 =						n_positions
								A_				 =						n_embd
								A_				 =						n_layer
								A_				 =						n_head
								A_				 =						n_inner
								A_				 =						rotary_dim
								A_				 =						activation_function
								A_				 =						resid_pdrop
								A_				 =						embd_pdrop
								A_				 =						attn_pdrop
								A_				 =						layer_norm_epsilon
								A_				 =						initializer_range
								A_				 =						use_cache

								A_				 =						bos_token_id
								A_				 =						eos_token_id

								super().__init__(
								    bos_token_id=_snake_case       ,   eos_token_id=_snake_case       ,   tie_word_embeddings=_snake_case       ,   **_snake_case	)





class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			def __init__(				self						:					Union[str, Any]       ,   _snake_case						:					PretrainedConfig       ,   _snake_case						:					str = "default"       ,   _snake_case						:					List[PatchingSpec] = None       ,   _snake_case						:					bool = False       ,   )      -> Optional[int]:







								"""simple docstring"""
								super().__init__(_snake_case       ,   task=_snake_case       ,   patching_specs=_snake_case       ,   use_past=_snake_case	)
								if not getattr(self._config       ,   "pad_token_id"       ,   _snake_case	):
													# TODO: how to do that better?
													A_				 =						0




			@property
			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Mapping[str, Mapping[int, str]]:







								"""simple docstring"""
								A_				 =						OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}	)
								if self.use_past:
													self.fill_with_past_key_values_(_snake_case       ,   direction="inputs"	)
													A_				 =						{0: "batch", 1: "past_sequence + sequence"}
								else:
													A_				 =						{0: "batch", 1: "sequence"}

								return common_inputs




			@property
			def      lowerCamelCase__     (				self						:					List[Any]	)      -> int:







								"""simple docstring"""
								return self._config.n_layer




			@property
			def      lowerCamelCase__     (				self						:					Any	)      -> int:







								"""simple docstring"""
								return self._config.n_head




			def      lowerCamelCase__     (				self						:					Optional[int]       ,   _snake_case						:					PreTrainedTokenizer       ,   _snake_case						:					int = -1       ,   _snake_case						:					int = -1       ,   _snake_case						:					bool = False       ,   _snake_case						:					Optional[TensorType] = None       ,   )      -> Mapping[str, Any]:







								"""simple docstring"""
								A_				 =						super(_snake_case       ,   self	).generate_dummy_inputs(
								    _snake_case       ,   batch_size=_snake_case       ,   seq_length=_snake_case       ,   is_pair=_snake_case       ,   framework=_snake_case	)

								# We need to order the input in the way they appears in the forward()
								A_				 =						OrderedDict({"input_ids": common_inputs["input_ids"]}	)

								# Need to add the past_keys
								if self.use_past:
													if not is_torch_available():
																		raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."	)
													else:
																		import torch

																		A_ ,    A_				 =						common_inputs["input_ids"].shape
																		# Not using the same length for past_key_values
																		A_				 =						seqlen + 2
																		A_				 =						(
																		    batch,
																		    self.num_attention_heads,
																		    past_key_values_length,
																		    self._config.hidden_size // self.num_attention_heads,
																		)
																		A_				 =						[
																		    (torch.zeros(_snake_case	), torch.zeros(_snake_case	)) for _ in range(self.num_layers	)
																		]

								A_				 =						common_inputs["attention_mask"]
								if self.use_past:
													A_				 =						ordered_inputs["attention_mask"].dtype
													A_				 =						torch.cat(
													    [ordered_inputs["attention_mask"], torch.ones(_snake_case       ,   _snake_case       ,   dtype=_snake_case	)]       ,   dim=1	)

								return ordered_inputs






			@property
			def      lowerCamelCase__     (				self						:					Dict	)      -> int:







								"""simple docstring"""
								return 13






"""simple docstring"""





import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class __lowerCAmelCase				(					_lowercase     ,				unittest.TestCase      ):




			"""simple docstring"""







			snake_case	      =				TransfoXLTokenizer
			snake_case	      =				False
			snake_case	      =				False




			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Optional[int]:







								"""simple docstring"""
								super().setUp()

								A_				 =						[
								    "<unk>",
								    "[CLS]",
								    "[SEP]",
								    "want",
								    "unwanted",
								    "wa",
								    "un",
								    "running",
								    ",",
								    "low",
								    "l",
								]
								A_				 =						os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES["vocab_file"]	)
								with open(self.vocab_file       ,   "w"       ,   encoding="utf-8"	) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)




			def      lowerCamelCase__     (				self						:					str       ,   **_snake_case						:					Any	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,   **_snake_case	)




			def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					Optional[Any]	)      -> Any:







								"""simple docstring"""
								A_				 =						"<unk> UNwanted , running"
								A_				 =						"<unk> unwanted, running"
								return input_text, output_text




			def      lowerCamelCase__     (				self						:					Dict	)      -> int:







								"""simple docstring"""
								A_				 =						TransfoXLTokenizer(vocab_file=self.vocab_file       ,   lower_case=_snake_case	)

								A_				 =						tokenizer.tokenize("<unk> UNwanted , running"	)
								self.assertListEqual(_snake_case       ,   ["<unk>", "unwanted", ",", "running"]	)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case	)       ,   [0, 4, 8, 7]	)




			def      lowerCamelCase__     (				self						:					List[str]	)      -> int:







								"""simple docstring"""
								A_				 =						TransfoXLTokenizer(lower_case=_snake_case	)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "	)       ,   ["hello", "!", "how", "are", "you", "?"]	)




			def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> str:







								"""simple docstring"""
								A_				 =						TransfoXLTokenizer(lower_case=_snake_case	)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "	)       ,   ["HeLLo", "!", "how", "Are", "yoU", "?"]	)




			def      lowerCamelCase__     (				self						:					Dict	)      -> Tuple:







								"""simple docstring"""
								A_				 =						TransfoXLTokenizer(lower_case=_snake_case	)
								A_				 =						"Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"
								A_				 =						[
								    "Hello",
								    "(",
								    "bracket",
								    ")",
								    "and",
								    "side",
								    "@-@",
								    "scrolled",
								    "[",
								    "and",
								    "]",
								    "Henry",
								    "'s",
								    "$",
								    "5",
								    "@,@",
								    "000",
								    "with",
								    "3",
								    "@.@",
								    "34",
								    "m",
								    ".",
								    "What",
								    "'s",
								    "up",
								    "!",
								    "?",
								]

								self.assertListEqual(tokenizer.tokenize(_snake_case	)       ,   _snake_case	)

								self.assertEqual(tokenizer.convert_tokens_to_string(_snake_case	)       ,   _snake_case	)






			def      lowerCamelCase__     (				self						:					List[Any]	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						self.get_tokenizer()
								A_				 =						len(_snake_case	)

								tokenizer.add_tokens(["new1", "new2"]	)
								tokenizer.move_added_token("new1"       ,   1	)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(_snake_case	)       ,   original_len + 2	)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode("new1"	)       ,   [1]	)
								self.assertEqual(tokenizer.decode([1]	)       ,   "new1"	)




