



from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:     Dict        =     logging.get_logger(__name__)

A_		:     Optional[int]        =     {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/config.json'''}



class 		_lowercase		(       UpperCAmelCase__	):
   _UpperCAmelCase     	=			'''openai-gpt'''
   _UpperCAmelCase     	=			{
       '''max_position_embeddings''': '''n_positions''',
       '''hidden_size''': '''n_embd''',
       '''num_attention_heads''': '''n_head''',
       '''num_hidden_layers''': '''n_layer''',
   }

   def __init__(      self			:       Dict							,   __lowerCAmelCase			:       Dict=4_0478							,   __lowerCAmelCase			:       Optional[int]=512							,   __lowerCAmelCase			:       int=768							,   __lowerCAmelCase			:       int=12							,   __lowerCAmelCase			:       Union[str, Any]=12							,   __lowerCAmelCase			:       List[str]="gelu"							,   __lowerCAmelCase			:       Union[str, Any]=0.1							,   __lowerCAmelCase			:       List[str]=0.1							,   __lowerCAmelCase			:       Union[str, Any]=0.1							,   __lowerCAmelCase			:       str=1E-5							,   __lowerCAmelCase			:       Optional[int]=0.0_2							,   __lowerCAmelCase			:       Optional[Any]="cls_index"							,   __lowerCAmelCase			:       List[Any]=True							,   __lowerCAmelCase			:       List[str]=None							,   __lowerCAmelCase			:       Any=True							,   __lowerCAmelCase			:       str=0.1							,   **__lowerCAmelCase			:       Dict							,   )       ->				List[Any]:


    """simple docstring"""






    a						=		vocab_size
    a						=		n_positions
    a						=		n_embd
    a						=		n_layer
    a						=		n_head
    a						=		afn
    a						=		resid_pdrop
    a						=		embd_pdrop
    a						=		attn_pdrop
    a						=		layer_norm_epsilon
    a						=		initializer_range
    a						=		summary_type
    a						=		summary_use_proj
    a						=		summary_activation
    a						=		summary_first_dropout
    a						=		summary_proj_to_labels
    super().__init__(**__lowerCAmelCase							)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:     str        =     logging.get_logger(__name__)

A_		:     List[Any]        =     {
    '''RWKV/rwkv-4-169m-pile''': '''https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-430m-pile''': '''https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-1b5-pile''': '''https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-3b-pile''': '''https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-7b-pile''': '''https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-14b-pile''': '''https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-raven-1b5''': '''https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json''',
    '''RWKV/rwkv-raven-3b''': '''https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-7b''': '''https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-14b''': '''https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json''',
}



class 		_lowercase		(       UpperCAmelCase__	):
   _UpperCAmelCase     	=			'''rwkv'''
   _UpperCAmelCase     	=			{'''max_position_embeddings''': '''context_length'''}

   def __init__(      self			:       List[str]							,   __lowerCAmelCase			:       Union[str, Any]=5_0277							,   __lowerCAmelCase			:       str=1024							,   __lowerCAmelCase			:       Union[str, Any]=4096							,   __lowerCAmelCase			:       Optional[int]=32							,   __lowerCAmelCase			:       Optional[int]=None							,   __lowerCAmelCase			:       Optional[Any]=None							,   __lowerCAmelCase			:       List[Any]=1E-5							,   __lowerCAmelCase			:       Union[str, Any]=0							,   __lowerCAmelCase			:       Optional[Any]=0							,   __lowerCAmelCase			:       Dict=6							,   __lowerCAmelCase			:       int=False							,   __lowerCAmelCase			:       Tuple=True							,   **__lowerCAmelCase			:       List[str]							,   )       ->				List[Any]:


    """simple docstring"""






    a						=		vocab_size
    a						=		context_length
    a						=		hidden_size
    a						=		num_hidden_layers
    a						=		attention_hidden_size if attention_hidden_size is not None else hidden_size
    a						=		intermediate_size if intermediate_size is not None else 4 * hidden_size
    a						=		layer_norm_epsilon
    a						=		rescale_every
    a						=		use_cache

    a						=		bos_token_id
    a						=		eos_token_id

    super().__init__(
        tie_word_embeddings=__lowerCAmelCase							,   bos_token_id=__lowerCAmelCase							,   eos_token_id=__lowerCAmelCase							,   **__lowerCAmelCase							)

