Update tokenizer.json
Browse files- tokenizer.json +21 -0
tokenizer.json
CHANGED
|
@@ -289,6 +289,27 @@
|
|
| 289 |
{
|
| 290 |
"type": "Fuse"
|
| 291 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
{
|
| 293 |
"type": "Replace",
|
| 294 |
"pattern": {
|
|
|
|
| 289 |
{
|
| 290 |
"type": "Fuse"
|
| 291 |
},
|
| 292 |
+
{
|
| 293 |
+
"type": "Replace",
|
| 294 |
+
"pattern": {
|
| 295 |
+
"String": "<|start_header_id|> "
|
| 296 |
+
},
|
| 297 |
+
"content": "<|start_header_id|>"
|
| 298 |
+
},
|
| 299 |
+
{
|
| 300 |
+
"type": "Replace",
|
| 301 |
+
"pattern": {
|
| 302 |
+
"String": "<|end_header_id|> "
|
| 303 |
+
},
|
| 304 |
+
"content": "<|end_header_id|>"
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"type": "Replace",
|
| 308 |
+
"pattern": {
|
| 309 |
+
"String": "<|eot_id|> "
|
| 310 |
+
},
|
| 311 |
+
"content": "<|eot_id|>"
|
| 312 |
+
},
|
| 313 |
{
|
| 314 |
"type": "Replace",
|
| 315 |
"pattern": {
|