Mistral tokenizer_config.json fix (#1)

- Mistral tokenizer_config.json fix (e0316eb32781334f9543d0b15aab28261967dd49)


Co-authored-by: Tom Jobbins <[email protected]>

tokenizer_config.json

@@ -4,25 +4,25 @@
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "1": {
       "content": "<s>",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "2": {
       "content": "</s>",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     }
@@ -37,9 +37,7 @@
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
-  "tokenizer_file": "/home/teknium/.cache/huggingface/hub/models--mistralai--Mistral-7B-v0.1/snapshots/ae9d75c6b4eb39515def78c685fb4d71d49fc2cf/tokenizer.json",
   "trust_remote_code": false,
   "unk_token": "<unk>",
   "use_default_system_prompt": true,
-  "use_fast": true
 }