



"""simple docstring"""





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCamelCase      =		logging.get_logger(__name__)


lowerCamelCase      =		{
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}
class    lowercase__     (					snake_case__   ):







       '''simple docstring'''


       UpperCamelCase		=  '''codegen'''
       UpperCamelCase		=  {
           '''max_position_embeddings''': '''n_positions''',
           '''hidden_size''': '''n_embd''',
           '''num_attention_heads''': '''n_head''',
           '''num_hidden_layers''': '''n_layer''',
       }



       def __init__(							self    :						int	,       _UpperCAmelCase    :						List[str]=50400	,       _UpperCAmelCase    :						Tuple=2048	,       _UpperCAmelCase    :						Any=2048	,       _UpperCAmelCase    :						Union[str, Any]=4096	,       _UpperCAmelCase    :						str=28	,       _UpperCAmelCase    :						Optional[int]=16	,       _UpperCAmelCase    :						List[Any]=64	,       _UpperCAmelCase    :						int=None	,       _UpperCAmelCase    :						List[str]="gelu_new"	,       _UpperCAmelCase    :						Optional[int]=0.0	,       _UpperCAmelCase    :						List[str]=0.0	,       _UpperCAmelCase    :						str=0.0	,       _UpperCAmelCase    :						List[str]=1e-5	,       _UpperCAmelCase    :						Tuple=0.02	,       _UpperCAmelCase    :						Optional[int]=True	,       _UpperCAmelCase    :						Optional[int]=50256	,       _UpperCAmelCase    :						str=50256	,       _UpperCAmelCase    :						List[str]=False	,       **_UpperCAmelCase    :						Optional[int]	,       )				->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							vocab_size
             UpperCAmelCase_    				=							n_ctx
             UpperCAmelCase_    				=							n_positions
             UpperCAmelCase_    				=							n_embd
             UpperCAmelCase_    				=							n_layer
             UpperCAmelCase_    				=							n_head
             UpperCAmelCase_    				=							n_inner
             UpperCAmelCase_    				=							rotary_dim
             UpperCAmelCase_    				=							activation_function
             UpperCAmelCase_    				=							resid_pdrop
             UpperCAmelCase_    				=							embd_pdrop
             UpperCAmelCase_    				=							attn_pdrop
             UpperCAmelCase_    				=							layer_norm_epsilon
             UpperCAmelCase_    				=							initializer_range
             UpperCAmelCase_    				=							use_cache

             UpperCAmelCase_    				=							bos_token_id
             UpperCAmelCase_    				=							eos_token_id

             super().__init__(
                 bos_token_id=_A	,       eos_token_id=_A	,       tie_word_embeddings=_A	,       **_A					)



class    lowercase__     (					snake_case__   ):







       '''simple docstring'''


       def __init__(							self    :						Union[str, Any]	,       _UpperCAmelCase    :						Dict	,       _UpperCAmelCase    :						List[Any] = "default"	,       _UpperCAmelCase    :						Union[str, Any] = None	,       _UpperCAmelCase    :						List[str] = False	,       )				->		Any:


             '''simple docstring'''




             super().__init__(_A	,       task=_A	,       patching_specs=_A	,       use_past=_A					)
             if not getattr(self._config	,       "pad_token_id"	,       _A					):
                   # TODO: how to do that better?
                   UpperCAmelCase_    				=							0



       @property
       def   lowercase__       (							self    :						str					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}					)
             if self.use_past:
                   self.fill_with_past_key_values_(_A	,       direction="inputs"					)
                   UpperCAmelCase_    				=							{0: 'batch', 1: 'past_sequence + sequence'}
             else:
                   UpperCAmelCase_    				=							{0: 'batch', 1: 'sequence'}

             return common_inputs



       @property
       def   lowercase__       (							self    :						Dict					)				->		Tuple:


             '''simple docstring'''




             return self._config.n_layer



       @property
       def   lowercase__       (							self    :						Any					)				->		Union[str, Any]:


             '''simple docstring'''




             return self._config.n_head



       def   lowercase__       (							self    :						str	,       _UpperCAmelCase    :						int	,       _UpperCAmelCase    :						Union[str, Any] = -1	,       _UpperCAmelCase    :						Dict = -1	,       _UpperCAmelCase    :						int = False	,       _UpperCAmelCase    :						Optional[Any] = None	,       )				->		str:


             '''simple docstring'''




             UpperCAmelCase_    				=							super(_A	,       self					).generate_dummy_inputs(
                 _A	,       batch_size=_A	,       seq_length=_A	,       is_pair=_A	,       framework=_A					)

             # We need to order the input in the way they appears in the forward()
             UpperCAmelCase_    				=							OrderedDict({"input_ids": common_inputs["input_ids"]}					)

             # Need to add the past_keys
             if self.use_past:
                   if not is_torch_available():
                         raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."					)
                   else:
                         import torch

                         UpperCAmelCase_    				=							common_inputs['input_ids'].shape
                         # Not using the same length for past_key_values
                         UpperCAmelCase_    				=							seqlen + 2
                         UpperCAmelCase_    				=							(
                             batch,
                             self.num_attention_heads,
                             past_key_values_length,
                             self._config.hidden_size // self.num_attention_heads,
                         )
                         UpperCAmelCase_    				=							[
                             (torch.zeros(_A					), torch.zeros(_A					)) for _ in range(self.num_layers					)
                         ]

             UpperCAmelCase_    				=							common_inputs['attention_mask']
             if self.use_past:
                   UpperCAmelCase_    				=							ordered_inputs['attention_mask'].dtype
                   UpperCAmelCase_    				=							torch.cat(
                       [ordered_inputs["attention_mask"], torch.ones(_A	,       _A	,       dtype=_A					)]	,       dim=1					)

             return ordered_inputs







       @property
       def   lowercase__       (							self    :						Union[str, Any]					)				->		Tuple:


             '''simple docstring'''




             return 13









"""simple docstring"""





import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      =		get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
class    lowercase__     (					SCREAMING_SNAKE_CASE       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  XLMProphetNetTokenizer
       UpperCamelCase		=  False
       UpperCamelCase		=  True



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Any:


             '''simple docstring'''




             super().setUp()

             # We have a SentencePiece fixture for testing
             UpperCAmelCase_    				=							XLMProphetNetTokenizer(_UpperCAmelCase	,       keep_accents=_UpperCAmelCase					)
             tokenizer.save_pretrained(self.tmpdirname					)



       def   lowercase__       (							self    :						Tuple					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							"[PAD]"
             UpperCAmelCase_    				=							0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase					)	,       _UpperCAmelCase					)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase					)	,       _UpperCAmelCase					)



       def   lowercase__       (							self    :						Dict					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							list(self.get_tokenizer().get_vocab().keys()					)

             self.assertEqual(vocab_keys[0]	,       "[PAD]"					)
             self.assertEqual(vocab_keys[1]	,       "[CLS]"					)
             self.assertEqual(vocab_keys[-1]	,       "j"					)
             self.assertEqual(len(_UpperCAmelCase					)	,       1012					)



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             self.assertEqual(self.get_tokenizer().vocab_size	,       1012					)



       def   lowercase__       (							self    :						Tuple					)				->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							XLMProphetNetTokenizer(_UpperCAmelCase	,       keep_accents=_UpperCAmelCase					)

             UpperCAmelCase_    				=							tokenizer.tokenize("This is a test"					)
             self.assertListEqual(_UpperCAmelCase	,       ["▁This", "▁is", "▁a", "▁t", "est"]					)

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)	,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,       )

             UpperCAmelCase_    				=							tokenizer.tokenize("I was born in 92000, and this is falsé."					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "9",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "é",
                     ".",
                 ]	,       )
             UpperCAmelCase_    				=							tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
                 ]	,       )

             UpperCAmelCase_    				=							tokenizer.convert_ids_to_tokens(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "[UNK]",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "[UNK]",
                     ".",
                 ]	,       )



       @cached_property
       def   lowercase__       (							self    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"					)



       @slow
       def   lowercase__       (							self    :						List[Any]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							"Hello World!"
             UpperCAmelCase_    				=							[35389, 6672, 49, 2]
             self.assertListEqual(_UpperCAmelCase	,       self.big_tokenizer.encode(_UpperCAmelCase					)					)







       @slow
       def   lowercase__       (							self    :						Optional[int]					)				->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_    				=							{"input_ids": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=_UpperCAmelCase	,       model_name="microsoft/xprophetnet-large-wiki100-cased"	,       revision="1acad1643ddd54a44df6a1b797ada8373685d90e"	,       )




