add tokenizer

tokenizer.json

@@ -129,67 +129,67 @@
       "y": 65,
       "z": 66,
       "##u": 67,
-      "##l": 68,
-      "##v": 69,
-      "##e": 70,
-      "##r": 71,
-      "##i": 72,
-      "##z": 73,
-      "##s": 74,
-      "##a": 75,
-      "##g": 76,
-      "##d": 77,
-      "##o": 78,
-      "##w": 79,
+      "##k": 68,
+      "##w": 69,
+      "##a": 70,
+      "##n": 71,
+      "##g": 72,
+      "##s": 73,
+      "##e": 74,
+      "##b": 75,
+      "##o": 76,
+      "##r": 77,
+      "##t": 78,
+      "##i": 79,
       "##m": 80,
-      "##n": 81,
-      "##t": 82,
-      "##p": 83,
-      "##k": 84,
-      "##b": 85,
-      "##T": 86,
-      "##R": 87,
-      "##I": 88,
-      "##N": 89,
-      "##8": 90,
-      "##9": 91,
-      "##0": 92,
-      "##h": 93,
-      "##c": 94,
-      "##f": 95,
-      "##5": 96,
-      "##3": 97,
-      "##y": 98,
-      "##2": 99,
-      "##1": 100,
-      "##x": 101,
-      "##7": 102,
-      "##6": 103,
-      "##4": 104,
-      "##S": 105,
-      "##D": 106,
-      "##C": 107,
-      "##j": 108,
-      "##A": 109,
-      "##F": 110,
+      "##d": 81,
+      "##x": 82,
+      "##O": 83,
+      "##l": 84,
+      "##c": 85,
+      "##v": 86,
+      "##A": 87,
+      "##C": 88,
+      "##2": 89,
+      "##R": 90,
+      "##1": 91,
+      "##h": 92,
+      "##j": 93,
+      "##p": 94,
+      "##S": 95,
+      "##y": 96,
+      "##7": 97,
+      "##6": 98,
+      "##5": 99,
+      "##0": 100,
+      "##9": 101,
+      "##4": 102,
+      "##3": 103,
+      "##E": 104,
+      "##L": 105,
+      "##M": 106,
+      "##D": 107,
+      "##U": 108,
+      "##B": 109,
+      "##f": 110,
       "##P": 111,
-      "##L": 112,
-      "##E": 113,
-      "##M": 114,
-      "##J": 115,
-      "##U": 116,
-      "##B": 117,
-      "##H": 118,
-      "##G": 119,
-      "##K": 120,
-      "##O": 121,
-      "##q": 122,
-      "##W": 123,
-      "##V": 124,
-      "##Y": 125,
+      "##K": 112,
+      "##G": 113,
+      "##T": 114,
+      "##Z": 115,
+      "##8": 116,
+      "##q": 117,
+      "##Y": 118,
+      "##W": 119,
+      "##V": 120,
+      "##I": 121,
+      "##F": 122,
+      "##H": 123,
+      "##z": 124,
+      "##N": 125,
       "##X": 126,
-      "##Q": 127,
-      "##Z": 128,
+      "##J": 127,
+      "##Q": 128,
       "##he": 129,
       "the": 130,
       "##on": 131,
@@ -5493,8 +5493,8 @@
       "colon": 5429,
       "depression": 5430,
       "##known": 5431,
-      "##men": 5432,
-      "##know": 5433,
+      "##know": 5432,
+      "##men": 5433,
       "##insula": 5434,
       "outdoor": 5435,
       "##unk": 5436,
@@ -9005,8 +9005,8 @@
       "IoT": 8941,
       "tac": 8942,
       "staying": 8943,
-      "##zens": 8944,
-      "##26": 8945,
+      "##26": 8944,
+      "##zens": 8945,
       "heated": 8946,
       "none": 8947,
       "temperatures": 8948,

tokenizer_config.json

@@ -1 +1 @@
-{"name_or_path": "english-WPE.json", "tokenizer_class": "PreTrainedTokenizerFast"}
+{"tokenizer_class": "PreTrainedTokenizerFast"}