| from transformers import PretrainedConfig | |
| class AveyConfig(PretrainedConfig): | |
| model_type = "avey" | |
| def __init__( | |
| self, | |
| vocab_size: int = 50281, | |
| d_embed: int = 768, | |
| n_blocks: int = 22, | |
| expansion_factor: int = 4, | |
| context_len: int = 256, | |
| **kwargs | |
| ): | |
| self.vocab_size = vocab_size | |
| self.d_embed = d_embed | |
| self.n_blocks = n_blocks | |
| self.expansion_factor = expansion_factor | |
| self.context_len = context_len | |
| super().__init__(**kwargs) | |