v2 of model

config.json

@@ -5,21 +5,21 @@
         "name": "gemma_backbone",
         "trainable": true,
         "vocabulary_size": 256000,
-        "num_layers": 18,
+        "num_layers": 26,
         "num_query_heads": 8,
-        "num_key_value_heads": 1,
-        "hidden_dim": 2048,
-        "intermediate_dim": 32768,
+        "num_key_value_heads": 4,
+        "hidden_dim": 2304,
+        "intermediate_dim": 18432,
         "head_dim": 256,
         "layer_norm_epsilon": 1e-06,
         "dropout": 0,
         "query_head_dim_normalize": true,
-        "use_post_ffw_norm": false,
-        "use_post_attention_norm": false,
-        "final_logit_soft_cap": null,
-        "attention_logit_soft_cap": null,
+        "use_post_ffw_norm": true,
+        "use_post_attention_norm": true,
+        "final_logit_soft_cap": 30.0,
+        "attention_logit_soft_cap": 50.0,
         "sliding_window_size": 4096,
-        "use_sliding_window_attention": false
+        "use_sliding_window_attention": true
     },
     "registered_name": "keras_hub>GemmaBackbone"
 }

metadata.json

@@ -1,6 +1,6 @@
 {
     "keras_version": "3.6.0",
     "keras_hub_version": "0.17.0",
-    "parameter_count": 2507536384,
-    "date_saved": "2024-11-20@00:28:07"
+    "parameter_count": 2617270528,
+    "date_saved": "2024-11-21@00:48:49"
 }

model.weights.h5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2d2ee8f20e0f098cb7b96363da64e9a56baf27544ad6525e75337a3ce7ffa39
-size 10025273304
+oid sha256:ffcd0d3ef03bcf2e7d810c328ccc6944ee0d08af98ad37b0b7fffa41607581d3
+size 10458345856

preprocessor.json

@@ -35,7 +35,7 @@
             "registered_name": "keras_hub>GemmaTokenizer"
         },
         "config_file": "preprocessor.json",
-        "sequence_length": 512,
+        "sequence_length": 64,
         "add_start_token": true,
         "add_end_token": true
     },

task.json

@@ -9,21 +9,21 @@
                 "name": "gemma_backbone",
                 "trainable": true,
                 "vocabulary_size": 256000,
-                "num_layers": 18,
+                "num_layers": 26,
                 "num_query_heads": 8,
-                "num_key_value_heads": 1,
-                "hidden_dim": 2048,
-                "intermediate_dim": 32768,
+                "num_key_value_heads": 4,
+                "hidden_dim": 2304,
+                "intermediate_dim": 18432,
                 "head_dim": 256,
                 "layer_norm_epsilon": 1e-06,
                 "dropout": 0,
                 "query_head_dim_normalize": true,
-                "use_post_ffw_norm": false,
-                "use_post_attention_norm": false,
-                "final_logit_soft_cap": null,
-                "attention_logit_soft_cap": null,
+                "use_post_ffw_norm": true,
+                "use_post_attention_norm": true,
+                "final_logit_soft_cap": 30.0,
+                "attention_logit_soft_cap": 50.0,
                 "sliding_window_size": 4096,
-                "use_sliding_window_attention": false
+                "use_sliding_window_attention": true
             },
             "registered_name": "keras_hub>GemmaBackbone"
         },
@@ -64,7 +64,7 @@
                     "registered_name": "keras_hub>GemmaTokenizer"
                 },
                 "config_file": "preprocessor.json",
-                "sequence_length": 512,
+                "sequence_length": 64,
                 "add_start_token": true,
                 "add_end_token": true
             },