Upload DogeForCausalLM

config.json

@@ -1,8 +1,8 @@
 {
-  "_name_or_path": "SmallDoge/Doge-60M",
   "architectures": [
     "DogeForCausalLM"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
   "auto_map": {
     "AutoConfig": "configuration_doge.DogeConfig",
@@ -19,14 +19,20 @@
   "initializer_range": 0.02,
   "intermediate_size": 1024,
   "is_moe": false,
+  "keep_window_size": 2048,
   "max_position_embeddings": 2048,
+  "mlp_bias": false,
   "model_type": "doge",
+  "norm_topk_prob": false,
   "num_attention_heads": 4,
   "num_cdmoe_experts": 16348,
   "num_cdmoe_experts_per_head": 8,
   "num_cdmoe_heads": 4,
+  "num_experts": 16384,
+  "num_experts_per_tok": 64,
   "num_hidden_layers": 16,
   "num_key_value_heads": 2,
+  "output_router_logits": false,
   "pad_token_id": 2,
   "rms_norm_eps": 1e-06,
   "rope_scaling": {
@@ -35,9 +41,11 @@
     "rope_type": "dynamic"
   },
   "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
   "tie_word_embeddings": true,
-  "torch_dtype": "float32",
-  "transformers_version": "4.48.3",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.0",
   "use_cache": true,
   "vocab_size": 32768
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 0,
   "eos_token_id": 1,
   "pad_token_id": 2,
-  "transformers_version": "4.48.3"
+  "transformers_version": "4.55.0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:550dbbf30bc9f8b88c7ac4136a1412414be8db29a5146b9f0bab2e795ab991e5
-size 218325576
+oid sha256:0c0974eb9bdd4c39eb840024424b9ca67186461d7ad8a7b414289cedd3f2289d
+size 109174080