jree423 commited on
Commit
97a0903
·
verified ·
1 Parent(s): 13ea232

Upload tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +21 -0
tokenizer.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [],
6
+ "normalizer": null,
7
+ "pre_tokenizer": null,
8
+ "post_processor": null,
9
+ "decoder": null,
10
+ "model": {
11
+ "type": "unigram",
12
+ "vocab": [
13
+ {
14
+ "id": 0,
15
+ "piece": "<unk>",
16
+ "score": 0.0
17
+ }
18
+ ],
19
+ "unk_id": 0
20
+ }
21
+ }