joelb/mistral-7b-instruct-ccai-scotland

README.md

@@ -35,17 +35,13 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
+- learning_rate: 4e-05
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- training_steps: 100
-
-### Training results
-
-
+- training_steps: 2500
 
 ### Framework versions
 

adapter_config.json

@@ -16,7 +16,7 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e44ce263e6fd885f50d82ca515b9325375b43ee36ededb75acf161ce88bc2e41
-size 48
+oid sha256:a570bc7c17a73dab9150e1228606feb00b120f6415f8fd4b05867884d2407925
+size 109069176

tokenizer_config.json

@@ -33,11 +33,15 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": false,
+  "max_length": 1024,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "</s>",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
+  "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4942072b2e9076eda35a8ae39ff7cda855315e7c4618163c1c3cfce0c10ffd54
+oid sha256:07dacbad5f929f75692c643f3fd2c20e49bfc7e66bac6419c5d0478e630be927
 size 4984