Upload folder using huggingface_hub

chat_template.jinja

@@ -0,0 +1 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' in message %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\n'}}{% endif %}

config.json

@@ -0,0 +1,72 @@
+{
+  "architectures": [
+    "DeepseekV3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_deepseek.DeepseekV3Config",
+    "AutoModel": "modeling_deepseek.DeepseekV3Model",
+    "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
+  },
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "ep_size": 1,
+  "first_k_dense_replace": 3,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 7168,
+  "initializer_range": 0.02,
+  "intermediate_size": 18432,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 163840,
+  "model_type": "deepseek_v3",
+  "moe_intermediate_size": 2048,
+  "moe_layer_freq": 1,
+  "n_group": 8,
+  "n_routed_experts": 256,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 128,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 128,
+  "num_nextn_predict_layers": 1,
+  "pretraining_tp": 1,
+  "q_lora_rank": 1536,
+  "qk_head_dim": 192,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 64,
+  "quantization_config": {
+    "activation_scheme": "dynamic",
+    "fmt": "e4m3",
+    "quant_method": "fp8",
+    "weight_block_size": [
+      128,
+      128
+    ]
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_interleave": true,
+  "rope_scaling": {
+    "beta_fast": 32.0,
+    "beta_slow": 1.0,
+    "factor": 40.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 4096,
+    "rope_type": "yarn",
+    "type": "yarn"
+  },
+  "rope_theta": 10000,
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "tie_word_embeddings": false,
+  "topk_group": 4,
+  "topk_method": "noaux_tc",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": true,
+  "v_head_dim": 128,
+  "vocab_size": 129280
+}

configuration_deepseek.py

@@ -0,0 +1,199 @@
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+class DeepseekV3Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`DeepseekV3Model`]. It is used to instantiate an DeepSeek
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the DeepSeek-V3.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 129280):
+            Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`DeepseekV3Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        moe_intermediate_size (`int`, *optional*, defaults to 1407):
+            Dimension of the MoE representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_nextn_predict_layers (`int`, *optional*, defaults to 1):
+            Number of nextn predict layers in the DeepSeekV3 Model.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        n_shared_experts (`int`, *optional*, defaults to None):
+            Number of shared experts, None means dense model.
+        n_routed_experts (`int`, *optional*, defaults to None):
+            Number of routed experts, None means dense model.
+        routed_scaling_factor (`float`, *optional*, defaults to 1.0):
+            Scaling factor or routed experts.
+        topk_method (`str`, *optional*, defaults to `gready`):
+            Topk method used in routed gate.
+        n_group (`int`, *optional*, defaults to None):
+            Number of groups for routed experts.
+        topk_group (`int`, *optional*, defaults to None):
+            Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
+        num_experts_per_tok (`int`, *optional*, defaults to None):
+            Number of selected experts, None means dense model.
+        moe_layer_freq (`int`, *optional*, defaults to 1):
+            The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
+        first_k_dense_replace (`int`, *optional*, defaults to 0):
+            Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
+                                                            \--k dense layers--/
+        norm_topk_prob (`bool`, *optional*, defaults to False):
+            Whether to normalize the weights of the routed experts.
+        scoring_func (`str`, *optional*, defaults to 'softmax'):
+            Method of computing expert weights.
+        aux_loss_alpha (`float`, *optional*, defaults to 0.001):
+            Auxiliary loss weight coefficient.
+        seq_aux = (`bool`, *optional*, defaults to True):
+            Whether to compute the auxiliary loss for each individual sample.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+            `max_position_embeddings` to the expected new maximum.
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    ```python
+    >>> from transformers import DeepseekV3Model, DeepseekV3Config
+
+    >>> # Initializing a Deepseek-V3 style configuration
+    >>> configuration = DeepseekV3Config()
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "deepseek_v3"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=129280,
+        hidden_size=7168,
+        intermediate_size=18432,
+        moe_intermediate_size = 2048,
+        num_hidden_layers=61,
+        num_nextn_predict_layers=1,
+        num_attention_heads=128,
+        num_key_value_heads=128,
+        n_shared_experts = 1,
+        n_routed_experts = 256,
+        ep_size = 1,
+        routed_scaling_factor = 2.5,
+        kv_lora_rank = 512,
+        q_lora_rank = 1536,
+        qk_rope_head_dim = 64,
+        v_head_dim = 128,
+        qk_nope_head_dim = 128,
+        topk_method = 'noaux_tc',
+        n_group = 8,
+        topk_group = 4,
+        num_experts_per_tok = 8,
+        moe_layer_freq = 1,
+        first_k_dense_replace = 3,
+        norm_topk_prob = True,
+        scoring_func = 'sigmoid',
+        hidden_act="silu",
+        max_position_embeddings=4096,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=0,
+        eos_token_id=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_nextn_predict_layers = num_nextn_predict_layers
+        self.num_attention_heads = num_attention_heads
+        self.n_shared_experts = n_shared_experts
+        self.n_routed_experts = n_routed_experts
+        self.ep_size = ep_size
+        self.routed_scaling_factor = routed_scaling_factor
+        self.kv_lora_rank = kv_lora_rank
+        self.q_lora_rank = q_lora_rank
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.v_head_dim = v_head_dim
+        self.qk_nope_head_dim = qk_nope_head_dim
+        self.topk_method = topk_method
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.num_experts_per_tok = num_experts_per_tok
+        self.moe_layer_freq = moe_layer_freq
+        self.first_k_dense_replace = first_k_dense_replace
+        self.norm_topk_prob = norm_topk_prob
+        self.scoring_func = scoring_func
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "transformers_version": "4.53.0"
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07a88fb4b94274fa4d6b770738bed4c6ade5b4f7530ab6fa60b0a85440cc4dfe
+size 4825751392

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dffab665851d5085174f8c53b15744ed1a4d81d997480452786728b1d52d1013
+size 4983801968

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c60cc29e358a7f30857e7b8cf67da93871da154265d62f7a22f7002a2f84c667
+size 4991242864

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2091e0ee20e861c16bb6003c34024dbdc2d27f62d8484596ff254858f441de30
+size 4991243024

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81f8d7530561b350fd12e7cbcb16990bbd21e59d0eb5f45333d85a499c9927bc
+size 4991243032

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ded685f8e99080e9365af3a9ba3f1cfd748a6763f91f64005f300a47d402d08
+size 3585664952

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbcf901a69be4dd1db9f280dfd876bf24642abdc02250c4da396f87a9266538a
+size 1853358208

model.safetensors.index.json

@@ -0,0 +1,829 @@
+{
+  "metadata": {
+    "total_parameters": 15111101440,
+    "total_size": 30222203392
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.kv_a_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.kv_b_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_a_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_a_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_b_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.kv_a_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.kv_b_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_a_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_a_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_b_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.self_attn.kv_a_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.kv_b_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_a_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_a_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_b_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.0.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.0.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.0.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.1.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.1.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.1.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.10.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.10.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.10.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.100.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.100.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.100.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.101.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.101.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.101.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.102.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.102.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.102.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.103.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.103.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.103.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.104.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.104.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.104.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.105.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.105.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.105.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.106.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.106.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.106.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.107.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.107.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.107.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.108.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.108.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.108.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.109.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.109.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.109.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.11.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.11.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.11.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.110.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.110.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.110.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.111.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.111.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.111.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.112.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.112.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.112.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.113.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.113.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.113.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.114.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.114.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.114.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.115.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.115.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.115.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.116.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.116.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.116.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.117.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.117.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.117.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.118.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.118.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.118.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.119.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.119.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.119.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.12.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.12.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.12.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.120.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.120.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.120.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.121.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.121.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.121.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.122.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.122.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.122.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.123.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.123.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.123.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.124.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.124.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.124.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.125.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.125.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.125.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.126.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.126.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.126.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.127.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.127.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.127.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.128.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.128.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.128.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.129.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.129.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.129.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.13.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.13.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.13.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.130.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.130.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.130.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.131.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.131.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.131.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.132.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.132.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.132.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.133.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.133.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.133.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.134.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.134.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.134.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.135.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.135.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.135.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.136.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.136.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.136.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.137.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.137.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.137.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.138.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.138.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.138.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.139.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.139.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.139.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.14.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.14.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.14.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.140.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.140.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.140.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.141.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.141.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.141.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.142.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.142.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.142.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.143.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.143.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.143.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.144.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.144.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.144.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.145.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.145.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.145.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.146.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.146.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.146.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.147.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.147.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.147.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.148.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.148.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.148.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.149.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.149.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.149.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.15.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.15.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.15.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.150.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.150.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.150.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.151.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.151.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.151.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.152.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.152.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.152.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.153.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.153.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.153.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.154.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.154.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.154.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.155.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.155.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.155.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.156.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.156.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.156.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.157.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.157.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.157.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.158.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.158.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.158.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.159.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.159.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.159.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.experts.16.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.16.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.16.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.160.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.160.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.160.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.161.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.161.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.161.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.162.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.162.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.162.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.163.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.163.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.163.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.164.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.164.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.164.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.165.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.165.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.165.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.166.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.166.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.166.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.167.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.167.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.167.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.168.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.168.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.168.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.169.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.169.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.169.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.17.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.17.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.17.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.170.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.170.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.170.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.171.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.171.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.171.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.172.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.172.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.172.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.173.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.173.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.173.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.174.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.174.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.174.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.175.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.175.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.175.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.176.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.176.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.176.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.177.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.177.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.177.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.178.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.178.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.178.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.179.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.179.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.179.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.18.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.18.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.18.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.180.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.180.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.180.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.181.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.181.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.181.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.182.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.182.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.182.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.183.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.183.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.183.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.184.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.184.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.184.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.185.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.185.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.185.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.186.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.186.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.186.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.187.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.187.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.187.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.188.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.188.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.188.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.189.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.189.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.189.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.19.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.19.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.19.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.190.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.190.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.190.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.191.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.191.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.191.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.192.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.192.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.192.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.193.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.193.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.193.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.194.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.194.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.194.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.195.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.195.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.195.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.196.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.196.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.196.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.197.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.197.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.197.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.198.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.198.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.198.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.199.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.199.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.199.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.2.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.2.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.2.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.20.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.20.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.20.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.200.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.200.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.200.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.201.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.201.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.201.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.202.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.202.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.202.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.203.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.203.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.203.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.204.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.204.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.204.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.205.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.205.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.205.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.206.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.206.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.206.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.207.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.207.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.207.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.208.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.208.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.208.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.209.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.209.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.209.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.21.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.21.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.21.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.210.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.210.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.210.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.211.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.211.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.211.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.212.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.212.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.212.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.213.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.213.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.213.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.214.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.214.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.214.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.215.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.215.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.215.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.216.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.216.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.mlp.experts.216.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.217.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.217.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.217.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.218.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.218.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.218.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.219.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.219.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.219.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.22.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.22.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.22.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.220.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.220.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.220.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.221.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.221.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.221.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.222.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.222.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.222.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.223.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.223.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.223.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.224.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.224.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.224.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.225.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.225.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.225.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.226.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.226.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.226.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.227.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.227.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.227.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.228.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.228.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.228.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.229.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.229.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.229.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.23.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.23.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.23.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.230.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.230.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.230.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.231.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.231.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.231.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.232.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.232.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.232.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.233.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.233.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.233.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.234.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.234.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.234.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.235.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.235.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.235.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.236.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.236.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.236.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.237.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.237.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.237.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.238.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.238.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.238.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.239.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.239.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.239.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.24.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.24.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.24.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.240.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.240.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.240.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.241.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.241.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.241.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.242.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.242.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.242.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.243.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.243.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.243.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.244.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.244.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.244.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.245.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.245.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.245.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.246.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.246.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.246.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.247.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.247.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.247.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.248.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.248.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.248.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.249.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.249.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.249.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.25.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.25.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.25.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.250.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.250.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.250.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.251.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.251.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.251.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.252.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.252.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.252.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.253.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.253.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.253.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.254.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.254.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.254.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.255.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.255.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.255.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.experts.26.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.26.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.26.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.27.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.27.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.27.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.28.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.28.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.28.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.29.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.29.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.29.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.3.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.3.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.3.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.30.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.30.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.30.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.31.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.31.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.31.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.32.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.32.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.32.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.33.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.33.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.33.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.34.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.34.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.34.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.35.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.35.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.35.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.36.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.36.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.36.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.37.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.37.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.37.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.38.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.38.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.38.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.39.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.39.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.39.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.4.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.4.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.4.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.40.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.40.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.40.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.41.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.41.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.41.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.42.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.42.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.42.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.43.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.43.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.43.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.44.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.44.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.44.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.45.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.45.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.45.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.46.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.46.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.46.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.47.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.47.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.47.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.48.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.48.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.48.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.49.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.49.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.49.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.5.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.5.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.5.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.50.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.50.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.50.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.51.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.51.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.51.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.52.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.52.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.52.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.53.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.53.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.53.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.54.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.54.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.54.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.55.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.55.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.55.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.56.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.56.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.56.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.57.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.57.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.57.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.58.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.58.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.58.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.59.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.59.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.59.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.6.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.6.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.6.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.60.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.60.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.60.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.61.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.61.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.61.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.62.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.62.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.62.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.63.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.63.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.63.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.64.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.64.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.64.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.65.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.65.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.65.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.66.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.66.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.66.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.67.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.67.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.67.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.68.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.68.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.68.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.69.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.69.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.69.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.7.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.7.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.7.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.70.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.70.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.70.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.71.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.71.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.71.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.72.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.72.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.72.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.73.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.73.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.73.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.74.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.74.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.74.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.75.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.75.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.75.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.76.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.76.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.76.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.77.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.77.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.77.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.78.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.78.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.78.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.79.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.79.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.79.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.8.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.8.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.8.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.80.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.80.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.80.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.81.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.81.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.81.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.82.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.82.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.82.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.83.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.83.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.83.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.84.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.84.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.84.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.85.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.85.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.85.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.86.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.86.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.86.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.87.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.87.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.87.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.88.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.88.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.88.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.89.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.89.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.89.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.9.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.9.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.9.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.experts.90.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.90.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.90.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.91.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.91.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.91.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.92.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.92.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.92.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.93.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.93.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.93.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.94.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.94.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.94.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.95.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.95.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.95.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.96.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.96.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.96.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.97.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.97.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.97.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.98.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.98.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.98.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.99.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.99.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.experts.99.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.3.mlp.gate.e_score_correction_bias": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.gate.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.self_attn.kv_a_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.kv_a_proj_with_mqa.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.kv_b_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_a_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_a_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_b_proj.weight": "model-00002-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}