Training in progress, step 1000, checkpoint

last-checkpoint/config.json

@@ -7,13 +7,13 @@
   "classifier_dropout": null,
   "dtype": "float32",
   "embedding_size": 128,
-  "expert_intermediate_size": 4096,
+  "expert_intermediate_size": 2624,
   "group_depth": 4,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 1024,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 2624,
+  "intermediate_size": 1152,
   "layer_norm_eps": 1e-06,
   "load_balancing_loss_coef": 0.2,
   "lora_alpha": 32,
@@ -21,9 +21,9 @@
   "max_position_embeddings": 8192,
   "model_type": "ModernALBERT",
   "num_attention_heads": 16,
-  "num_expert_modules": 4,
+  "num_expert_modules": 2,
   "num_experts": 8,
-  "num_hidden_layers": 24,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
   "router_jitter_noise": 0.01,
   "top_k": 2,

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80119511b3945f8aecd2aff333fcc080ae35d401fa55f8697938e14ea5789293
-size 715030586
+oid sha256:3981c224c2a7d945f7fe3e53c0fb9d4e7407abfb83a401e7824cb1133ea82245
+size 189211642

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db0934189152b7908f6b35dae2253e40f6b3aded20fa897418941775fe3b4cbc
-size 1032262338
+oid sha256:dffd333913e4ed94e5ed1309111e2f93752cf719ce33e2ad74841fef8a730e2d
+size 363608098

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:034eca388b6e0fac986e87c89720add477525fe2b57f1b529dbea803344c29a2
+oid sha256:9d5fabadabdfa20a4193a06d044902f5ac084cc01ec1c50a1345b5b99385fd5f
 size 14960

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db42f786a33e3872b59b1ca275f25b90d7b69af2b0c2b79b4d7fd5d70559dc7a
+oid sha256:e9511bae90cb3d5a4ff03189808c05515b151b1d77964e7f713734be9042277c
 size 14960

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a36d0cf7ab0f4d122469dbd181cba437132f6365ffad1a565ff07188961a6ae
+oid sha256:10ec5da6115b06a543f49f097ef2f42f6e7c13c60917114815a5f8431718cabc
 size 14960

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9258a401eddff240d305cd10dc17c44142df9febe7dfe2abf5b20a0a6ae19d15
+oid sha256:80eefeb08928a05000827d723367e88a43d18becfa46957ea485357d24f91a78
 size 14960

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f916885680464b362867980a30f750ed0f39325c0710cedfa4270e808cfb4d4
+oid sha256:5140203923e7ee9e99429beb6bab58c04e3c2e04f95473c908703ae1e156d413
 size 1064

last-checkpoint/tokenizer.json

@@ -2,7 +2,7 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 4048,
+    "max_length": 1024,
     "strategy": "LongestFirst",
     "stride": 0
   },

last-checkpoint/tokenizer_config.json

@@ -937,7 +937,7 @@
     "input_ids",
     "attention_mask"
   ],
-  "model_max_length": 4048,
+  "model_max_length": 1024,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
   "tokenizer_class": "PreTrainedTokenizer",

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aedb58df69b659a844d0320b7629d8ada1f48db758fcb589f094136e2177e3b0
+oid sha256:1f7ab0ffa64a712076054007d1389d5f4c75c80b03fcc26405d9947d274385f9
 size 5432