





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_lowerCamelCase   				=			logging.get_logger(__name__)

_lowerCamelCase   				=			{
    'EleutherAI/gpt-j-6B': 'https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json',
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}




class   UpperCamelCase_       (     UpperCamelCase__    ):
							lowerCamelCase_           =					"gptj"
							lowerCamelCase_           =					{
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}


							def __init__(							self    :Dict							,				__A    :Optional[int]=5_0400							,				__A    :List[str]=2048							,				__A    :List[str]=4096							,				__A    :Any=28							,				__A    :Tuple=16							,				__A    :Any=64							,				__A    :List[str]=None							,				__A    :List[Any]="gelu_new"							,				__A    :int=0.0							,				__A    :str=0.0							,				__A    :int=0.0							,				__A    :int=1E-5							,				__A    :str=0.0_2							,				__A    :Union[str, Any]=True							,				__A    :Optional[Any]=5_0256							,				__A    :Dict=5_0256							,				__A    :Dict=False							,				**__A    :Dict							,				)   ->   Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      vocab_size
									SCREAMING_SNAKE_CASE__			=      n_positions
									SCREAMING_SNAKE_CASE__			=      n_embd
									SCREAMING_SNAKE_CASE__			=      n_layer
									SCREAMING_SNAKE_CASE__			=      n_head
									SCREAMING_SNAKE_CASE__			=      n_inner
									SCREAMING_SNAKE_CASE__			=      rotary_dim
									SCREAMING_SNAKE_CASE__			=      activation_function
									SCREAMING_SNAKE_CASE__			=      resid_pdrop
									SCREAMING_SNAKE_CASE__			=      embd_pdrop
									SCREAMING_SNAKE_CASE__			=      attn_pdrop
									SCREAMING_SNAKE_CASE__			=      layer_norm_epsilon
									SCREAMING_SNAKE_CASE__			=      initializer_range
									SCREAMING_SNAKE_CASE__			=      use_cache

									SCREAMING_SNAKE_CASE__			=      bos_token_id
									SCREAMING_SNAKE_CASE__			=      eos_token_id

									super().__init__(
									    bos_token_id=__A							,				eos_token_id=__A							,				tie_word_embeddings=__A							,				**__A		)




class   UpperCamelCase_       (     UpperCamelCase__    ):


							def __init__(							self    :List[Any]							,				__A    :PretrainedConfig							,				__A    :str = "default"							,				__A    :List[PatchingSpec] = None							,				__A    :bool = False							,				)   ->   List[str]:



									"""simple docstring"""



									super().__init__(__A							,				task=__A							,				patching_specs=__A							,				use_past=__A		)
									if not getattr(self._config							,				"""pad_token_id"""							,				__A		):
											# TODO: how to do that better?
											SCREAMING_SNAKE_CASE__			=      0


							@property
							def      _snake_case  (							self    :Optional[Any]		)   ->   Mapping[str, Mapping[int, str]]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}		)
									if self.use_past:
											self.fill_with_past_key_values_(__A							,				direction="""inputs"""		)
											SCREAMING_SNAKE_CASE__			=      {0: """batch""", 1: """past_sequence + sequence"""}
									else:
											SCREAMING_SNAKE_CASE__			=      {0: """batch""", 1: """sequence"""}

									return common_inputs


							@property
							def      _snake_case  (							self    :Dict		)   ->   int:



									"""simple docstring"""



									return self._config.n_layer


							@property
							def      _snake_case  (							self    :str		)   ->   int:



									"""simple docstring"""



									return self._config.n_head


							def      _snake_case  (							self    :str							,				__A    :PreTrainedTokenizer							,				__A    :int = -1							,				__A    :int = -1							,				__A    :bool = False							,				__A    :Optional[TensorType] = None							,				)   ->   Mapping[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      super(__A							,				self		).generate_dummy_inputs(
									    __A							,				batch_size=__A							,				seq_length=__A							,				is_pair=__A							,				framework=__A		)

									# We need to order the input in the way they appears in the forward()
									SCREAMING_SNAKE_CASE__			=      OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}		)

									# Need to add the past_keys
									if self.use_past:
											if not is_torch_available():
													raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""		)
											else:
													import torch

													SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      common_inputs["""input_ids"""].shape
													# Not using the same length for past_key_values
													SCREAMING_SNAKE_CASE__			=      seqlen + 2
													SCREAMING_SNAKE_CASE__			=      (
													    batch,
													    self.num_attention_heads,
													    past_key_values_length,
													    self._config.hidden_size // self.num_attention_heads,
													)
													SCREAMING_SNAKE_CASE__			=      [
													    (torch.zeros(__A		), torch.zeros(__A		)) for _ in range(self.num_layers		)
													]

									SCREAMING_SNAKE_CASE__			=      common_inputs["""attention_mask"""]
									if self.use_past:
											SCREAMING_SNAKE_CASE__			=      ordered_inputs["""attention_mask"""].dtype
											SCREAMING_SNAKE_CASE__			=      torch.cat(
											    [ordered_inputs["""attention_mask"""], torch.ones(__A							,				__A							,				dtype=__A		)]							,				dim=1		)

									return ordered_inputs


							@property
							def      _snake_case  (							self    :List[Any]		)   ->   int:



									"""simple docstring"""



									return 13






import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					OpenAIGPTTokenizer
							lowerCamelCase_           =					OpenAIGPTTokenizerFast
							lowerCamelCase_           =					True
							lowerCamelCase_           =					False


							def      _snake_case  (							self    :Optional[Any]		)   ->   Dict:



									"""simple docstring"""



									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									SCREAMING_SNAKE_CASE__			=      [
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """w</w>""",
									    """r</w>""",
									    """t</w>""",
									    """lo""",
									    """low""",
									    """er</w>""",
									    """low</w>""",
									    """lowest</w>""",
									    """newer</w>""",
									    """wider</w>""",
									    """<unk>""",
									]
									SCREAMING_SNAKE_CASE__			=      dict(zip(__A							,				range(len(__A		)		)		)		)
									SCREAMING_SNAKE_CASE__			=      ["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""merges_file"""]		)
									with open(self.vocab_file							,				"""w"""		) as fp:
											fp.write(json.dumps(__A		)		)
									with open(self.merges_file							,				"""w"""		) as fp:
											fp.write("""\n""".join(__A		)		)


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :str		)   ->   List[Any]:



									"""simple docstring"""



									return "lower newer", "lower newer"


							def      _snake_case  (							self    :Optional[Any]		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      OpenAIGPTTokenizer(self.vocab_file							,				self.merges_file		)

									SCREAMING_SNAKE_CASE__			=      """lower"""
									SCREAMING_SNAKE_CASE__			=      ["""low""", """er</w>"""]
									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
									self.assertListEqual(__A							,				__A		)

									SCREAMING_SNAKE_CASE__			=      tokens + ["""<unk>"""]
									SCREAMING_SNAKE_CASE__			=      [14, 15, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				__A		)


							def      _snake_case  (							self    :Optional[Any]							,				__A    :Optional[Any]=15		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													# Simple input
													SCREAMING_SNAKE_CASE__			=      """This is a simple input"""
													SCREAMING_SNAKE_CASE__			=      ["""This is a simple input 1""", """This is a simple input 2"""]
													SCREAMING_SNAKE_CASE__			=      ("""This is a simple input""", """This is a pair""")
													SCREAMING_SNAKE_CASE__			=      [
													    ("""This is a simple input 1""", """This is a simple input 2"""),
													    ("""This is a simple pair 1""", """This is a simple pair 2"""),
													]

													# Simple input tests
													self.assertRaises(__A							,				tokenizer_r.encode							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Simple input
													self.assertRaises(__A							,				tokenizer_r.encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Simple input
													self.assertRaises(
													    __A							,				tokenizer_r.batch_encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""							,				)

													# Pair input
													self.assertRaises(__A							,				tokenizer_r.encode							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Pair input
													self.assertRaises(__A							,				tokenizer_r.encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Pair input
													self.assertRaises(
													    __A							,				tokenizer_r.batch_encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""							,				)


							def      _snake_case  (							self    :Dict		)   ->   List[Any]:



									"""simple docstring"""



									pass




@require_ftfy
@require_spacy
@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__    ):
							pass