Commit
·
fd34c39
1
Parent(s):
0bf38c0
add tokenizer
Browse files- vocab.json +1 -1
vocab.json
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
{"'": 13, "a": 14, "b": 4, "c": 7, "d": 20, "e": 15, "f": 11, "g": 19, "h": 18, "i": 5, "j": 10, "k": 0, "l": 16, "m": 24, "n": 6, "o": 25, "p": 27, "q": 12, "r": 9, "s": 21, "t": 17, "u": 2, "v": 8, "w": 3, "x": 23, "y": 1, "z": 22, "|": 26, "[UNK]":
|
|
|
|
| 1 |
+
{"-": 29, "_": 28, "'": 13, "a": 14, "b": 4, "c": 7, "d": 20, "e": 15, "f": 11, "g": 19, "h": 18, "i": 5, "j": 10, "k": 0, "l": 16, "m": 24, "n": 6, "o": 25, "p": 27, "q": 12, "r": 9, "s": 21, "t": 17, "u": 2, "v": 8, "w": 3, "x": 23, "y": 1, "z": 22, "|": 26, "[UNK]": 30, "[PAD]": 31}
|