DavidAU commited on
Commit
ffe72ea
·
verified ·
1 Parent(s): 4ab3ee8

Upload config.json

Browse files
Files changed (1) hide show
  1. config.json +6 -2
config.json CHANGED
@@ -12,7 +12,7 @@
12
  "hidden_size": 2048,
13
  "initializer_range": 0.02,
14
  "intermediate_size": 6144,
15
- "max_position_embeddings": 40960,
16
  "max_window_layers": 48,
17
  "mlp_only_layers": [],
18
  "model_type": "qwen3_moe",
@@ -25,7 +25,11 @@
25
  "num_key_value_heads": 4,
26
  "output_router_logits": false,
27
  "rms_norm_eps": 1e-06,
28
- "rope_scaling": null,
 
 
 
 
29
  "rope_theta": 1000000.0,
30
  "router_aux_loss_coef": 0.001,
31
  "sliding_window": null,
 
12
  "hidden_size": 2048,
13
  "initializer_range": 0.02,
14
  "intermediate_size": 6144,
15
+ "max_position_embeddings": 131072,
16
  "max_window_layers": 48,
17
  "mlp_only_layers": [],
18
  "model_type": "qwen3_moe",
 
25
  "num_key_value_heads": 4,
26
  "output_router_logits": false,
27
  "rms_norm_eps": 1e-06,
28
+ "rope_scaling": {
29
+ "type": "yarn",
30
+ "factor": 4.0,
31
+ "original_max_position_embeddings": 32768
32
+ },
33
  "rope_theta": 1000000.0,
34
  "router_aux_loss_coef": 0.001,
35
  "sliding_window": null,