yarn scale to 122k context length

Please don't merge or close this. I'm just going to use this pr revision to run the model at 122k sequence length

config.json

@@ -25,7 +25,11 @@
   "num_key_value_heads": 4,
   "output_router_logits": false,
   "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
+  "rope_scaling": {
+        "rope_type": "yarn",
+        "factor": 3.0,
+        "original_max_position_embeddings": 40960
+  },
   "rope_theta": 1000000.0,
   "router_aux_loss_coef": 0.001,
   "sliding_window": null,