Hugging Face's logo

allenai
/
GraspMolmo

Robotics
Safetensors
English
molmo
grasping
task-oriented-grasping
manipulation
custom_code
Model card Files
xet
Community
1
Abhay D commited on
Commit
c170777
·
1 Parent(s): 4ef0fdf

Add model files

Browse files
Files changed (23) hide show
  1. .gitattributes +1 -0
  2. added_tokens.json +428 -0
  3. config.json +72 -0
  4. config_molmo.py +154 -0
  5. generation_config.json +4 -0
  6. image_preprocessing_molmo.py +559 -0
  7. merges.txt +0 -0
  8. model-00001-of-00007.safetensors +3 -0
  9. model-00002-of-00007.safetensors +3 -0
  10. model-00003-of-00007.safetensors +3 -0
  11. model-00004-of-00007.safetensors +3 -0
  12. model-00005-of-00007.safetensors +3 -0
  13. model-00006-of-00007.safetensors +3 -0
  14. model-00007-of-00007.safetensors +3 -0
  15. model.safetensors.index.json +592 -0
  16. modeling_molmo.py +1409 -0
  17. preprocessing_molmo.py +189 -0
  18. preprocessor_config.json +22 -0
  19. processor_config.json +6 -0
  20. special_tokens_map.json +441 -0
  21. tokenizer.json +3 -0
  22. tokenizer_config.json +3853 -0
  23. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,428 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<im_col>": 152067,
3
+ "<im_end>": 152065,
4
+ "<im_patch>": 152066,
5
+ "<im_start>": 152064,
6
+ "<|endoftext|>": 151643,
7
+ "<|im_end|>": 151645,
8
+ "<|im_start|>": 151644,
9
+ "<|image|>": 152068,
10
+ "|<EXTRA_TOKENS_0>|": 151646,
11
+ "|<EXTRA_TOKENS_100>|": 151746,
12
+ "|<EXTRA_TOKENS_101>|": 151747,
13
+ "|<EXTRA_TOKENS_102>|": 151748,
14
+ "|<EXTRA_TOKENS_103>|": 151749,
15
+ "|<EXTRA_TOKENS_104>|": 151750,
16
+ "|<EXTRA_TOKENS_105>|": 151751,
17
+ "|<EXTRA_TOKENS_106>|": 151752,
18
+ "|<EXTRA_TOKENS_107>|": 151753,
19
+ "|<EXTRA_TOKENS_108>|": 151754,
20
+ "|<EXTRA_TOKENS_109>|": 151755,
21
+ "|<EXTRA_TOKENS_10>|": 151656,
22
+ "|<EXTRA_TOKENS_110>|": 151756,
23
+ "|<EXTRA_TOKENS_111>|": 151757,
24
+ "|<EXTRA_TOKENS_112>|": 151758,
25
+ "|<EXTRA_TOKENS_113>|": 151759,
26
+ "|<EXTRA_TOKENS_114>|": 151760,
27
+ "|<EXTRA_TOKENS_115>|": 151761,
28
+ "|<EXTRA_TOKENS_116>|": 151762,
29
+ "|<EXTRA_TOKENS_117>|": 151763,
30
+ "|<EXTRA_TOKENS_118>|": 151764,
31
+ "|<EXTRA_TOKENS_119>|": 151765,
32
+ "|<EXTRA_TOKENS_11>|": 151657,
33
+ "|<EXTRA_TOKENS_120>|": 151766,
34
+ "|<EXTRA_TOKENS_121>|": 151767,
35
+ "|<EXTRA_TOKENS_122>|": 151768,
36
+ "|<EXTRA_TOKENS_123>|": 151769,
37
+ "|<EXTRA_TOKENS_124>|": 151770,
38
+ "|<EXTRA_TOKENS_125>|": 151771,
39
+ "|<EXTRA_TOKENS_126>|": 151772,
40
+ "|<EXTRA_TOKENS_127>|": 151773,
41
+ "|<EXTRA_TOKENS_128>|": 151774,
42
+ "|<EXTRA_TOKENS_129>|": 151775,
43
+ "|<EXTRA_TOKENS_12>|": 151658,
44
+ "|<EXTRA_TOKENS_130>|": 151776,
45
+ "|<EXTRA_TOKENS_131>|": 151777,
46
+ "|<EXTRA_TOKENS_132>|": 151778,
47
+ "|<EXTRA_TOKENS_133>|": 151779,
48
+ "|<EXTRA_TOKENS_134>|": 151780,
49
+ "|<EXTRA_TOKENS_135>|": 151781,
50
+ "|<EXTRA_TOKENS_136>|": 151782,
51
+ "|<EXTRA_TOKENS_137>|": 151783,
52
+ "|<EXTRA_TOKENS_138>|": 151784,
53
+ "|<EXTRA_TOKENS_139>|": 151785,
54
+ "|<EXTRA_TOKENS_13>|": 151659,
55
+ "|<EXTRA_TOKENS_140>|": 151786,
56
+ "|<EXTRA_TOKENS_141>|": 151787,
57
+ "|<EXTRA_TOKENS_142>|": 151788,
58
+ "|<EXTRA_TOKENS_143>|": 151789,
59
+ "|<EXTRA_TOKENS_144>|": 151790,
60
+ "|<EXTRA_TOKENS_145>|": 151791,
61
+ "|<EXTRA_TOKENS_146>|": 151792,
62
+ "|<EXTRA_TOKENS_147>|": 151793,
63
+ "|<EXTRA_TOKENS_148>|": 151794,
64
+ "|<EXTRA_TOKENS_149>|": 151795,
65
+ "|<EXTRA_TOKENS_14>|": 151660,
66
+ "|<EXTRA_TOKENS_150>|": 151796,
67
+ "|<EXTRA_TOKENS_151>|": 151797,
68
+ "|<EXTRA_TOKENS_152>|": 151798,
69
+ "|<EXTRA_TOKENS_153>|": 151799,
70
+ "|<EXTRA_TOKENS_154>|": 151800,
71
+ "|<EXTRA_TOKENS_155>|": 151801,
72
+ "|<EXTRA_TOKENS_156>|": 151802,
73
+ "|<EXTRA_TOKENS_157>|": 151803,
74
+ "|<EXTRA_TOKENS_158>|": 151804,
75
+ "|<EXTRA_TOKENS_159>|": 151805,
76
+ "|<EXTRA_TOKENS_15>|": 151661,
77
+ "|<EXTRA_TOKENS_160>|": 151806,
78
+ "|<EXTRA_TOKENS_161>|": 151807,
79
+ "|<EXTRA_TOKENS_162>|": 151808,
80
+ "|<EXTRA_TOKENS_163>|": 151809,
81
+ "|<EXTRA_TOKENS_164>|": 151810,
82
+ "|<EXTRA_TOKENS_165>|": 151811,
83
+ "|<EXTRA_TOKENS_166>|": 151812,
84
+ "|<EXTRA_TOKENS_167>|": 151813,
85
+ "|<EXTRA_TOKENS_168>|": 151814,
86
+ "|<EXTRA_TOKENS_169>|": 151815,
87
+ "|<EXTRA_TOKENS_16>|": 151662,
88
+ "|<EXTRA_TOKENS_170>|": 151816,
89
+ "|<EXTRA_TOKENS_171>|": 151817,
90
+ "|<EXTRA_TOKENS_172>|": 151818,
91
+ "|<EXTRA_TOKENS_173>|": 151819,
92
+ "|<EXTRA_TOKENS_174>|": 151820,
93
+ "|<EXTRA_TOKENS_175>|": 151821,
94
+ "|<EXTRA_TOKENS_176>|": 151822,
95
+ "|<EXTRA_TOKENS_177>|": 151823,
96
+ "|<EXTRA_TOKENS_178>|": 151824,
97
+ "|<EXTRA_TOKENS_179>|": 151825,
98
+ "|<EXTRA_TOKENS_17>|": 151663,
99
+ "|<EXTRA_TOKENS_180>|": 151826,
100
+ "|<EXTRA_TOKENS_181>|": 151827,
101
+ "|<EXTRA_TOKENS_182>|": 151828,
102
+ "|<EXTRA_TOKENS_183>|": 151829,
103
+ "|<EXTRA_TOKENS_184>|": 151830,
104
+ "|<EXTRA_TOKENS_185>|": 151831,
105
+ "|<EXTRA_TOKENS_186>|": 151832,
106
+ "|<EXTRA_TOKENS_187>|": 151833,
107
+ "|<EXTRA_TOKENS_188>|": 151834,
108
+ "|<EXTRA_TOKENS_189>|": 151835,
109
+ "|<EXTRA_TOKENS_18>|": 151664,
110
+ "|<EXTRA_TOKENS_190>|": 151836,
111
+ "|<EXTRA_TOKENS_191>|": 151837,
112
+ "|<EXTRA_TOKENS_192>|": 151838,
113
+ "|<EXTRA_TOKENS_193>|": 151839,
114
+ "|<EXTRA_TOKENS_194>|": 151840,
115
+ "|<EXTRA_TOKENS_195>|": 151841,
116
+ "|<EXTRA_TOKENS_196>|": 151842,
117
+ "|<EXTRA_TOKENS_197>|": 151843,
118
+ "|<EXTRA_TOKENS_198>|": 151844,
119
+ "|<EXTRA_TOKENS_199>|": 151845,
120
+ "|<EXTRA_TOKENS_19>|": 151665,
121
+ "|<EXTRA_TOKENS_1>|": 151647,
122
+ "|<EXTRA_TOKENS_200>|": 151846,
123
+ "|<EXTRA_TOKENS_201>|": 151847,
124
+ "|<EXTRA_TOKENS_202>|": 151848,
125
+ "|<EXTRA_TOKENS_203>|": 151849,
126
+ "|<EXTRA_TOKENS_204>|": 151850,
127
+ "|<EXTRA_TOKENS_205>|": 151851,
128
+ "|<EXTRA_TOKENS_206>|": 151852,
129
+ "|<EXTRA_TOKENS_207>|": 151853,
130
+ "|<EXTRA_TOKENS_208>|": 151854,
131
+ "|<EXTRA_TOKENS_209>|": 151855,
132
+ "|<EXTRA_TOKENS_20>|": 151666,
133
+ "|<EXTRA_TOKENS_210>|": 151856,
134
+ "|<EXTRA_TOKENS_211>|": 151857,
135
+ "|<EXTRA_TOKENS_212>|": 151858,
136
+ "|<EXTRA_TOKENS_213>|": 151859,
137
+ "|<EXTRA_TOKENS_214>|": 151860,
138
+ "|<EXTRA_TOKENS_215>|": 151861,
139
+ "|<EXTRA_TOKENS_216>|": 151862,
140
+ "|<EXTRA_TOKENS_217>|": 151863,
141
+ "|<EXTRA_TOKENS_218>|": 151864,
142
+ "|<EXTRA_TOKENS_219>|": 151865,
143
+ "|<EXTRA_TOKENS_21>|": 151667,
144
+ "|<EXTRA_TOKENS_220>|": 151866,
145
+ "|<EXTRA_TOKENS_221>|": 151867,
146
+ "|<EXTRA_TOKENS_222>|": 151868,
147
+ "|<EXTRA_TOKENS_223>|": 151869,
148
+ "|<EXTRA_TOKENS_224>|": 151870,
149
+ "|<EXTRA_TOKENS_225>|": 151871,
150
+ "|<EXTRA_TOKENS_226>|": 151872,
151
+ "|<EXTRA_TOKENS_227>|": 151873,
152
+ "|<EXTRA_TOKENS_228>|": 151874,
153
+ "|<EXTRA_TOKENS_229>|": 151875,
154
+ "|<EXTRA_TOKENS_22>|": 151668,
155
+ "|<EXTRA_TOKENS_230>|": 151876,
156
+ "|<EXTRA_TOKENS_231>|": 151877,
157
+ "|<EXTRA_TOKENS_232>|": 151878,
158
+ "|<EXTRA_TOKENS_233>|": 151879,
159
+ "|<EXTRA_TOKENS_234>|": 151880,
160
+ "|<EXTRA_TOKENS_235>|": 151881,
161
+ "|<EXTRA_TOKENS_236>|": 151882,
162
+ "|<EXTRA_TOKENS_237>|": 151883,
163
+ "|<EXTRA_TOKENS_238>|": 151884,
164
+ "|<EXTRA_TOKENS_239>|": 151885,
165
+ "|<EXTRA_TOKENS_23>|": 151669,
166
+ "|<EXTRA_TOKENS_240>|": 151886,
167
+ "|<EXTRA_TOKENS_241>|": 151887,
168
+ "|<EXTRA_TOKENS_242>|": 151888,
169
+ "|<EXTRA_TOKENS_243>|": 151889,
170
+ "|<EXTRA_TOKENS_244>|": 151890,
171
+ "|<EXTRA_TOKENS_245>|": 151891,
172
+ "|<EXTRA_TOKENS_246>|": 151892,
173
+ "|<EXTRA_TOKENS_247>|": 151893,
174
+ "|<EXTRA_TOKENS_248>|": 151894,
175
+ "|<EXTRA_TOKENS_249>|": 151895,
176
+ "|<EXTRA_TOKENS_24>|": 151670,
177
+ "|<EXTRA_TOKENS_250>|": 151896,
178
+ "|<EXTRA_TOKENS_251>|": 151897,
179
+ "|<EXTRA_TOKENS_252>|": 151898,
180
+ "|<EXTRA_TOKENS_253>|": 151899,
181
+ "|<EXTRA_TOKENS_254>|": 151900,
182
+ "|<EXTRA_TOKENS_255>|": 151901,
183
+ "|<EXTRA_TOKENS_256>|": 151902,
184
+ "|<EXTRA_TOKENS_257>|": 151903,
185
+ "|<EXTRA_TOKENS_258>|": 151904,
186
+ "|<EXTRA_TOKENS_259>|": 151905,
187
+ "|<EXTRA_TOKENS_25>|": 151671,
188
+ "|<EXTRA_TOKENS_260>|": 151906,
189
+ "|<EXTRA_TOKENS_261>|": 151907,
190
+ "|<EXTRA_TOKENS_262>|": 151908,
191
+ "|<EXTRA_TOKENS_263>|": 151909,
192
+ "|<EXTRA_TOKENS_264>|": 151910,
193
+ "|<EXTRA_TOKENS_265>|": 151911,
194
+ "|<EXTRA_TOKENS_266>|": 151912,
195
+ "|<EXTRA_TOKENS_267>|": 151913,
196
+ "|<EXTRA_TOKENS_268>|": 151914,
197
+ "|<EXTRA_TOKENS_269>|": 151915,
198
+ "|<EXTRA_TOKENS_26>|": 151672,
199
+ "|<EXTRA_TOKENS_270>|": 151916,
200
+ "|<EXTRA_TOKENS_271>|": 151917,
201
+ "|<EXTRA_TOKENS_272>|": 151918,
202
+ "|<EXTRA_TOKENS_273>|": 151919,
203
+ "|<EXTRA_TOKENS_274>|": 151920,
204
+ "|<EXTRA_TOKENS_275>|": 151921,
205
+ "|<EXTRA_TOKENS_276>|": 151922,
206
+ "|<EXTRA_TOKENS_277>|": 151923,
207
+ "|<EXTRA_TOKENS_278>|": 151924,
208
+ "|<EXTRA_TOKENS_279>|": 151925,
209
+ "|<EXTRA_TOKENS_27>|": 151673,
210
+ "|<EXTRA_TOKENS_280>|": 151926,
211
+ "|<EXTRA_TOKENS_281>|": 151927,
212
+ "|<EXTRA_TOKENS_282>|": 151928,
213
+ "|<EXTRA_TOKENS_283>|": 151929,
214
+ "|<EXTRA_TOKENS_284>|": 151930,
215
+ "|<EXTRA_TOKENS_285>|": 151931,
216
+ "|<EXTRA_TOKENS_286>|": 151932,
217
+ "|<EXTRA_TOKENS_287>|": 151933,
218
+ "|<EXTRA_TOKENS_288>|": 151934,
219
+ "|<EXTRA_TOKENS_289>|": 151935,
220
+ "|<EXTRA_TOKENS_28>|": 151674,
221
+ "|<EXTRA_TOKENS_290>|": 151936,
222
+ "|<EXTRA_TOKENS_291>|": 151937,
223
+ "|<EXTRA_TOKENS_292>|": 151938,
224
+ "|<EXTRA_TOKENS_293>|": 151939,
225
+ "|<EXTRA_TOKENS_294>|": 151940,
226
+ "|<EXTRA_TOKENS_295>|": 151941,
227
+ "|<EXTRA_TOKENS_296>|": 151942,
228
+ "|<EXTRA_TOKENS_297>|": 151943,
229
+ "|<EXTRA_TOKENS_298>|": 151944,
230
+ "|<EXTRA_TOKENS_299>|": 151945,
231
+ "|<EXTRA_TOKENS_29>|": 151675,
232
+ "|<EXTRA_TOKENS_2>|": 151648,
233
+ "|<EXTRA_TOKENS_300>|": 151946,
234
+ "|<EXTRA_TOKENS_301>|": 151947,
235
+ "|<EXTRA_TOKENS_302>|": 151948,
236
+ "|<EXTRA_TOKENS_303>|": 151949,
237
+ "|<EXTRA_TOKENS_304>|": 151950,
238
+ "|<EXTRA_TOKENS_305>|": 151951,
239
+ "|<EXTRA_TOKENS_306>|": 151952,
240
+ "|<EXTRA_TOKENS_307>|": 151953,
241
+ "|<EXTRA_TOKENS_308>|": 151954,
242
+ "|<EXTRA_TOKENS_309>|": 151955,
243
+ "|<EXTRA_TOKENS_30>|": 151676,
244
+ "|<EXTRA_TOKENS_310>|": 151956,
245
+ "|<EXTRA_TOKENS_311>|": 151957,
246
+ "|<EXTRA_TOKENS_312>|": 151958,
247
+ "|<EXTRA_TOKENS_313>|": 151959,
248
+ "|<EXTRA_TOKENS_314>|": 151960,
249
+ "|<EXTRA_TOKENS_315>|": 151961,
250
+ "|<EXTRA_TOKENS_316>|": 151962,
251
+ "|<EXTRA_TOKENS_317>|": 151963,
252
+ "|<EXTRA_TOKENS_318>|": 151964,
253
+ "|<EXTRA_TOKENS_319>|": 151965,
254
+ "|<EXTRA_TOKENS_31>|": 151677,
255
+ "|<EXTRA_TOKENS_320>|": 151966,
256
+ "|<EXTRA_TOKENS_321>|": 151967,
257
+ "|<EXTRA_TOKENS_322>|": 151968,
258
+ "|<EXTRA_TOKENS_323>|": 151969,
259
+ "|<EXTRA_TOKENS_324>|": 151970,
260
+ "|<EXTRA_TOKENS_325>|": 151971,
261
+ "|<EXTRA_TOKENS_326>|": 151972,
262
+ "|<EXTRA_TOKENS_327>|": 151973,
263
+ "|<EXTRA_TOKENS_328>|": 151974,
264
+ "|<EXTRA_TOKENS_329>|": 151975,
265
+ "|<EXTRA_TOKENS_32>|": 151678,
266
+ "|<EXTRA_TOKENS_330>|": 151976,
267
+ "|<EXTRA_TOKENS_331>|": 151977,
268
+ "|<EXTRA_TOKENS_332>|": 151978,
269
+ "|<EXTRA_TOKENS_333>|": 151979,
270
+ "|<EXTRA_TOKENS_334>|": 151980,
271
+ "|<EXTRA_TOKENS_335>|": 151981,
272
+ "|<EXTRA_TOKENS_336>|": 151982,
273
+ "|<EXTRA_TOKENS_337>|": 151983,
274
+ "|<EXTRA_TOKENS_338>|": 151984,
275
+ "|<EXTRA_TOKENS_339>|": 151985,
276
+ "|<EXTRA_TOKENS_33>|": 151679,
277
+ "|<EXTRA_TOKENS_340>|": 151986,
278
+ "|<EXTRA_TOKENS_341>|": 151987,
279
+ "|<EXTRA_TOKENS_342>|": 151988,
280
+ "|<EXTRA_TOKENS_343>|": 151989,
281
+ "|<EXTRA_TOKENS_344>|": 151990,
282
+ "|<EXTRA_TOKENS_345>|": 151991,
283
+ "|<EXTRA_TOKENS_346>|": 151992,
284
+ "|<EXTRA_TOKENS_347>|": 151993,
285
+ "|<EXTRA_TOKENS_348>|": 151994,
286
+ "|<EXTRA_TOKENS_349>|": 151995,
287
+ "|<EXTRA_TOKENS_34>|": 151680,
288
+ "|<EXTRA_TOKENS_350>|": 151996,
289
+ "|<EXTRA_TOKENS_351>|": 151997,
290
+ "|<EXTRA_TOKENS_352>|": 151998,
291
+ "|<EXTRA_TOKENS_353>|": 151999,
292
+ "|<EXTRA_TOKENS_354>|": 152000,
293
+ "|<EXTRA_TOKENS_355>|": 152001,
294
+ "|<EXTRA_TOKENS_356>|": 152002,
295
+ "|<EXTRA_TOKENS_357>|": 152003,
296
+ "|<EXTRA_TOKENS_358>|": 152004,
297
+ "|<EXTRA_TOKENS_359>|": 152005,
298
+ "|<EXTRA_TOKENS_35>|": 151681,
299
+ "|<EXTRA_TOKENS_360>|": 152006,
300
+ "|<EXTRA_TOKENS_361>|": 152007,
301
+ "|<EXTRA_TOKENS_362>|": 152008,
302
+ "|<EXTRA_TOKENS_363>|": 152009,
303
+ "|<EXTRA_TOKENS_364>|": 152010,
304
+ "|<EXTRA_TOKENS_365>|": 152011,
305
+ "|<EXTRA_TOKENS_366>|": 152012,
306
+ "|<EXTRA_TOKENS_367>|": 152013,
307
+ "|<EXTRA_TOKENS_368>|": 152014,
308
+ "|<EXTRA_TOKENS_369>|": 152015,
309
+ "|<EXTRA_TOKENS_36>|": 151682,
310
+ "|<EXTRA_TOKENS_370>|": 152016,
311
+ "|<EXTRA_TOKENS_371>|": 152017,
312
+ "|<EXTRA_TOKENS_372>|": 152018,
313
+ "|<EXTRA_TOKENS_373>|": 152019,
314
+ "|<EXTRA_TOKENS_374>|": 152020,
315
+ "|<EXTRA_TOKENS_375>|": 152021,
316
+ "|<EXTRA_TOKENS_376>|": 152022,
317
+ "|<EXTRA_TOKENS_377>|": 152023,
318
+ "|<EXTRA_TOKENS_378>|": 152024,
319
+ "|<EXTRA_TOKENS_379>|": 152025,
320
+ "|<EXTRA_TOKENS_37>|": 151683,
321
+ "|<EXTRA_TOKENS_380>|": 152026,
322
+ "|<EXTRA_TOKENS_381>|": 152027,
323
+ "|<EXTRA_TOKENS_382>|": 152028,
324
+ "|<EXTRA_TOKENS_383>|": 152029,
325
+ "|<EXTRA_TOKENS_384>|": 152030,
326
+ "|<EXTRA_TOKENS_385>|": 152031,
327
+ "|<EXTRA_TOKENS_386>|": 152032,
328
+ "|<EXTRA_TOKENS_387>|": 152033,
329
+ "|<EXTRA_TOKENS_388>|": 152034,
330
+ "|<EXTRA_TOKENS_389>|": 152035,
331
+ "|<EXTRA_TOKENS_38>|": 151684,
332
+ "|<EXTRA_TOKENS_390>|": 152036,
333
+ "|<EXTRA_TOKENS_391>|": 152037,
334
+ "|<EXTRA_TOKENS_392>|": 152038,
335
+ "|<EXTRA_TOKENS_393>|": 152039,
336
+ "|<EXTRA_TOKENS_394>|": 152040,
337
+ "|<EXTRA_TOKENS_395>|": 152041,
338
+ "|<EXTRA_TOKENS_396>|": 152042,
339
+ "|<EXTRA_TOKENS_397>|": 152043,
340
+ "|<EXTRA_TOKENS_398>|": 152044,
341
+ "|<EXTRA_TOKENS_399>|": 152045,
342
+ "|<EXTRA_TOKENS_39>|": 151685,
343
+ "|<EXTRA_TOKENS_3>|": 151649,
344
+ "|<EXTRA_TOKENS_400>|": 152046,
345
+ "|<EXTRA_TOKENS_401>|": 152047,
346
+ "|<EXTRA_TOKENS_402>|": 152048,
347
+ "|<EXTRA_TOKENS_403>|": 152049,
348
+ "|<EXTRA_TOKENS_404>|": 152050,
349
+ "|<EXTRA_TOKENS_405>|": 152051,
350
+ "|<EXTRA_TOKENS_406>|": 152052,
351
+ "|<EXTRA_TOKENS_407>|": 152053,
352
+ "|<EXTRA_TOKENS_408>|": 152054,
353
+ "|<EXTRA_TOKENS_409>|": 152055,
354
+ "|<EXTRA_TOKENS_40>|": 151686,
355
+ "|<EXTRA_TOKENS_410>|": 152056,
356
+ "|<EXTRA_TOKENS_411>|": 152057,
357
+ "|<EXTRA_TOKENS_412>|": 152058,
358
+ "|<EXTRA_TOKENS_413>|": 152059,
359
+ "|<EXTRA_TOKENS_414>|": 152060,
360
+ "|<EXTRA_TOKENS_415>|": 152061,
361
+ "|<EXTRA_TOKENS_416>|": 152062,
362
+ "|<EXTRA_TOKENS_417>|": 152063,
363
+ "|<EXTRA_TOKENS_41>|": 151687,
364
+ "|<EXTRA_TOKENS_42>|": 151688,
365
+ "|<EXTRA_TOKENS_43>|": 151689,
366
+ "|<EXTRA_TOKENS_44>|": 151690,
367
+ "|<EXTRA_TOKENS_45>|": 151691,
368
+ "|<EXTRA_TOKENS_46>|": 151692,
369
+ "|<EXTRA_TOKENS_47>|": 151693,
370
+ "|<EXTRA_TOKENS_48>|": 151694,
371
+ "|<EXTRA_TOKENS_49>|": 151695,
372
+ "|<EXTRA_TOKENS_4>|": 151650,
373
+ "|<EXTRA_TOKENS_50>|": 151696,
374
+ "|<EXTRA_TOKENS_51>|": 151697,
375
+ "|<EXTRA_TOKENS_52>|": 151698,
376
+ "|<EXTRA_TOKENS_53>|": 151699,
377
+ "|<EXTRA_TOKENS_54>|": 151700,
378
+ "|<EXTRA_TOKENS_55>|": 151701,
379
+ "|<EXTRA_TOKENS_56>|": 151702,
380
+ "|<EXTRA_TOKENS_57>|": 151703,
381
+ "|<EXTRA_TOKENS_58>|": 151704,
382
+ "|<EXTRA_TOKENS_59>|": 151705,
383
+ "|<EXTRA_TOKENS_5>|": 151651,
384
+ "|<EXTRA_TOKENS_60>|": 151706,
385
+ "|<EXTRA_TOKENS_61>|": 151707,
386
+ "|<EXTRA_TOKENS_62>|": 151708,
387
+ "|<EXTRA_TOKENS_63>|": 151709,
388
+ "|<EXTRA_TOKENS_64>|": 151710,
389
+ "|<EXTRA_TOKENS_65>|": 151711,
390
+ "|<EXTRA_TOKENS_66>|": 151712,
391
+ "|<EXTRA_TOKENS_67>|": 151713,
392
+ "|<EXTRA_TOKENS_68>|": 151714,
393
+ "|<EXTRA_TOKENS_69>|": 151715,
394
+ "|<EXTRA_TOKENS_6>|": 151652,
395
+ "|<EXTRA_TOKENS_70>|": 151716,
396
+ "|<EXTRA_TOKENS_71>|": 151717,
397
+ "|<EXTRA_TOKENS_72>|": 151718,
398
+ "|<EXTRA_TOKENS_73>|": 151719,
399
+ "|<EXTRA_TOKENS_74>|": 151720,
400
+ "|<EXTRA_TOKENS_75>|": 151721,
401
+ "|<EXTRA_TOKENS_76>|": 151722,
402
+ "|<EXTRA_TOKENS_77>|": 151723,
403
+ "|<EXTRA_TOKENS_78>|": 151724,
404
+ "|<EXTRA_TOKENS_79>|": 151725,
405
+ "|<EXTRA_TOKENS_7>|": 151653,
406
+ "|<EXTRA_TOKENS_80>|": 151726,
407
+ "|<EXTRA_TOKENS_81>|": 151727,
408
+ "|<EXTRA_TOKENS_82>|": 151728,
409
+ "|<EXTRA_TOKENS_83>|": 151729,
410
+ "|<EXTRA_TOKENS_84>|": 151730,
411
+ "|<EXTRA_TOKENS_85>|": 151731,
412
+ "|<EXTRA_TOKENS_86>|": 151732,
413
+ "|<EXTRA_TOKENS_87>|": 151733,
414
+ "|<EXTRA_TOKENS_88>|": 151734,
415
+ "|<EXTRA_TOKENS_89>|": 151735,
416
+ "|<EXTRA_TOKENS_8>|": 151654,
417
+ "|<EXTRA_TOKENS_90>|": 151736,
418
+ "|<EXTRA_TOKENS_91>|": 151737,
419
+ "|<EXTRA_TOKENS_92>|": 151738,
420
+ "|<EXTRA_TOKENS_93>|": 151739,
421
+ "|<EXTRA_TOKENS_94>|": 151740,
422
+ "|<EXTRA_TOKENS_95>|": 151741,
423
+ "|<EXTRA_TOKENS_96>|": 151742,
424
+ "|<EXTRA_TOKENS_97>|": 151743,
425
+ "|<EXTRA_TOKENS_98>|": 151744,
426
+ "|<EXTRA_TOKENS_99>|": 151745,
427
+ "|<EXTRA_TOKENS_9>|": 151655
428
+ }
config.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_type": "silu",
3
+ "additional_vocab_size": 128,
4
+ "architectures": [
5
+ "MolmoForCausalLM"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "attention_type": "sdpa",
9
+ "auto_map": {
10
+ "AutoConfig": "config_molmo.MolmoConfig",
11
+ "AutoModelForCausalLM": "modeling_molmo.MolmoForCausalLM"
12
+ },
13
+ "bias_for_layer_norm": false,
14
+ "clip_qkv": null,
15
+ "embedding_dropout": 0.0,
16
+ "embedding_size": 152064,
17
+ "float32_attention": true,
18
+ "hidden_size": 3584,
19
+ "image_feature_dropout": 0.0,
20
+ "image_padding_embed": "pad_and_partial_pad",
21
+ "initializer_range": 0.02,
22
+ "intermediate_size": 37888,
23
+ "layer_norm_eps": 1e-06,
24
+ "layer_norm_type": "rms",
25
+ "max_position_embeddings": 4096,
26
+ "model_type": "molmo",
27
+ "moe_num_experts": 0,
28
+ "moe_top_k": 2,
29
+ "norm_after": false,
30
+ "normalize_input_embeds": false,
31
+ "num_attention_heads": 28,
32
+ "num_hidden_layers": 28,
33
+ "num_key_value_heads": 4,
34
+ "qk_layer_norm": false,
35
+ "qkv_bias": true,
36
+ "residual_dropout": 0.0,
37
+ "rope_theta": 1000000.0,
38
+ "scale_logits": false,
39
+ "tie_word_embeddings": false,
40
+ "torch_dtype": "float32",
41
+ "transformers_version": "4.50.3",
42
+ "use_cache": true,
43
+ "use_position_ids": true,
44
+ "vision_config": {
45
+ "attention_dropout": 0.0,
46
+ "float32_attention": true,
47
+ "image_default_input_size": [
48
+ 336,
49
+ 336
50
+ ],
51
+ "image_emb_dim": 1024,
52
+ "image_head_dim": 64,
53
+ "image_mlp_activations": "quick_gelu",
54
+ "image_mlp_dim": 4096,
55
+ "image_norm_eps": 1e-05,
56
+ "image_num_heads": 16,
57
+ "image_num_key_value_heads": 16,
58
+ "image_num_layers": 23,
59
+ "image_num_pos": 577,
60
+ "image_patch_size": 14,
61
+ "image_pos_patch_size": 14,
62
+ "initializer_range": 0.02,
63
+ "model_type": "",
64
+ "residual_dropout": 0.0
65
+ },
66
+ "vit_layers": [
67
+ -2,
68
+ -9
69
+ ],
70
+ "vocab_size": 152064,
71
+ "weight_tying": false
72
+ }
config_molmo.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List, Tuple
2
+
3
+ from transformers import PretrainedConfig, AutoTokenizer
4
+
5
+
6
+ class MolmoVisionConfig(PretrainedConfig):
7
+ def __init__(
8
+ self,
9
+ image_default_input_size: Tuple[int, int] = (336, 336),
10
+ image_patch_size: int = 14,
11
+ image_pos_patch_size: int = 14,
12
+ image_emb_dim: int = 1024,
13
+ image_num_heads: int = 16,
14
+ image_num_key_value_heads: int = 16,
15
+ image_num_layers: int = 23,
16
+ image_head_dim: int = 64,
17
+ image_mlp_dim: int = 4096,
18
+ image_mlp_activations: str = "quick_gelu",
19
+ residual_dropout: float = 0,
20
+ image_num_pos: int = 577,
21
+ image_norm_eps: float = 1e-5,
22
+ float32_attention: bool = True,
23
+ attention_type: str = "spda",
24
+ **kwargs
25
+ ):
26
+ super().__init__(**kwargs)
27
+ self.image_default_input_size = image_default_input_size
28
+ self.image_patch_size = image_patch_size
29
+ self.image_pos_patch_size = image_pos_patch_size
30
+ self.image_emb_dim = image_emb_dim
31
+ self.image_num_heads = image_num_heads
32
+ self.image_num_key_value_heads = image_num_key_value_heads
33
+ self.image_num_layers = image_num_layers
34
+ self.image_head_dim = image_head_dim
35
+ self.image_mlp_dim = image_mlp_dim
36
+ self.image_mlp_activations = image_mlp_activations
37
+ self.residual_dropout = residual_dropout
38
+ self.image_num_pos = image_num_pos
39
+ self.image_norm_eps = image_norm_eps
40
+ self.float32_attention = float32_attention
41
+
42
+ @property
43
+ def image_num_patch(self):
44
+ h, w = self.image_default_input_size
45
+ return h // self.image_patch_size, w // self.image_patch_size
46
+
47
+
48
+ class MolmoConfig(PretrainedConfig):
49
+ model_type = "molmo"
50
+ keys_to_ignore_at_inference = ["past_key_values"]
51
+
52
+ def __init__(
53
+ self,
54
+ vocab_size=50304,
55
+ embedding_size=50304,
56
+ hidden_size=4096,
57
+ intermediate_size=11008,
58
+ num_hidden_layers=32,
59
+ num_attention_heads=32,
60
+ num_key_value_heads=None,
61
+ float32_attention=True,
62
+ max_position_embeddings=2048,
63
+ initializer_range=0.02,
64
+ use_cache=True,
65
+ layer_norm_eps: float = 1e-5,
66
+ rope_theta=10000.0,
67
+ clip_qkv=None,
68
+ activation_type="silu",
69
+ qkv_bias: bool = False,
70
+ weight_tying: bool = False,
71
+ use_position_ids: bool=True,
72
+ tie_word_embeddings: bool=True,
73
+ bias_for_layer_norm: bool=False,
74
+ qk_layer_norm: bool=False,
75
+ norm_after: bool = False,
76
+ layer_norm_type: str="rms",
77
+ vision_config: MolmoVisionConfig=None,
78
+ vit_layers=(-2, -9),
79
+ residual_dropout: float=0.0,
80
+ embedding_dropout: float=0.0,
81
+ attention_dropout: float=0.0,
82
+ image_feature_dropout: float=0.0,
83
+ additional_vocab_size=128,
84
+ attention_type: str = "sdpa",
85
+ image_padding_embed="pad_and_partial_pad",
86
+ moe_num_experts=None,
87
+ moe_top_k=None,
88
+ normalize_input_embeds: bool=False,
89
+ scale_logits: bool=False,
90
+ **kwargs,
91
+ ):
92
+ if isinstance(vision_config, dict):
93
+ self.vision_config = MolmoVisionConfig(**vision_config)
94
+ elif vision_config is None:
95
+ self.vision_config = MolmoVisionConfig()
96
+ else:
97
+ self.vision_config = vision_config
98
+
99
+ self.vocab_size = vocab_size
100
+ self.embedding_size = embedding_size
101
+ self.max_position_embeddings = max_position_embeddings
102
+ self.hidden_size = hidden_size
103
+ self.intermediate_size = intermediate_size
104
+ self.num_hidden_layers = num_hidden_layers
105
+ self.num_attention_heads = num_attention_heads
106
+ self.layer_norm_eps = layer_norm_eps
107
+ self.weight_tying = weight_tying
108
+ self.use_position_ids = use_position_ids
109
+ self.qk_layer_norm = qk_layer_norm
110
+ self.num_key_value_heads = num_key_value_heads
111
+ self.float32_attention= float32_attention
112
+ self.initializer_range = initializer_range
113
+ self.use_cache = use_cache
114
+ self.rope_theta = rope_theta
115
+ self.clip_qkv = clip_qkv
116
+ self.activation_type = activation_type
117
+ self.qkv_bias = qkv_bias
118
+ self.norm_after = norm_after
119
+ self.tie_word_embeddings = tie_word_embeddings
120
+ self.layer_norm_type = layer_norm_type
121
+ self.moe_num_experts = moe_num_experts
122
+ self.moe_top_k = moe_top_k
123
+ self.vit_layers = vit_layers
124
+ self.residual_dropout = residual_dropout
125
+ self.embedding_dropout = embedding_dropout
126
+ self.attention_dropout = attention_dropout
127
+ self.image_feature_dropout = image_feature_dropout
128
+ self.image_padding_embed = image_padding_embed
129
+ self.bias_for_layer_norm = bias_for_layer_norm
130
+ self.additional_vocab_size = additional_vocab_size
131
+ self.attention_type = attention_type
132
+ self.normalize_input_embeds = normalize_input_embeds
133
+ self.scale_logits = scale_logits
134
+
135
+ super().__init__(
136
+ tie_word_embeddings=tie_word_embeddings,
137
+ **kwargs,
138
+ )
139
+
140
+ @property
141
+ def effective_num_key_value_heads(self) -> int:
142
+ if self.num_key_value_heads is None:
143
+ return self.num_attention_heads
144
+ else:
145
+ return self.num_key_value_heads
146
+
147
+ @property
148
+ def image_num_patch(self):
149
+ assert self.vision_config is not None
150
+ return self.vision_config.image_num_patch
151
+
152
+
153
+ MolmoVisionConfig.register_for_auto_class()
154
+ MolmoConfig.register_for_auto_class()
generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "transformers_version": "4.50.3"
4
+ }
image_preprocessing_molmo.py ADDED
@@ -0,0 +1,559 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 FIXME copyright?
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """Image processor class for Molmo"""
16
+ import pdb
17
+ from typing import List, Optional, Union, Mapping
18
+
19
+ import numpy as np
20
+ import torch
21
+ import torchvision.transforms
22
+ from torchvision.transforms import InterpolationMode
23
+ from torchvision.transforms.functional import convert_image_dtype
24
+
25
+ from transformers.image_utils import (
26
+ OPENAI_CLIP_MEAN,
27
+ OPENAI_CLIP_STD,
28
+ ImageInput,
29
+ )
30
+ from transformers.processing_utils import ImagesKwargs
31
+ from transformers.image_processing_utils import BaseImageProcessor
32
+ from transformers.utils import logging
33
+
34
+
35
+ logger = logging.get_logger(__name__)
36
+
37
+
38
+ def resize_and_pad(
39
+ image,
40
+ desired_output_size,
41
+ resize_method="torch-bilinear",
42
+ pad_value=0,
43
+ normalize=True,
44
+ image_mean=OPENAI_CLIP_MEAN,
45
+ image_std=OPENAI_CLIP_STD,
46
+ ):
47
+ """Resize an image while padding to preserve uts aspect ratio."""
48
+ desired_height, desired_width = desired_output_size
49
+ height, width = image.shape[:2]
50
+
51
+ # Cast into float32 since the training code did this in float32 and it (very rarely) effects
52
+ # the results after rounding.
53
+ image_scale_y = np.array(desired_height, np.float32) / np.array(height, np.float32)
54
+ image_scale_x = np.array(desired_width, np.float32) / np.array(width, np.float32)
55
+ image_scale = min(image_scale_x, image_scale_y)
56
+ scaled_height = int(np.array(height, np.float32) * image_scale)
57
+ scaled_width = int(np.array(width, np.float32) * image_scale)
58
+
59
+ if resize_method == "tensorflow":
60
+ # This how the original training code did resizing, it can produce slightly different
61
+ # results then using torch resize so we keep it just in case
62
+ import tensorflow as tf
63
+ image = tf.image.convert_image_dtype(tf.constant(image), dtype=tf.float32)
64
+ image = tf.image.resize(
65
+ image,
66
+ [scaled_height, scaled_width],
67
+ method=tf.image.ResizeMethod.BILINEAR,
68
+ antialias=True,
69
+ )
70
+ image = tf.clip_by_value(image, 0.0, 1.0)
71
+ image = image.numpy()
72
+ elif resize_method == "torch-bilinear":
73
+ image = torch.permute(torch.from_numpy(image), [2, 0, 1])
74
+ image = convert_image_dtype(image) # resize in float32 to match the training code
75
+ image = torchvision.transforms.Resize(
76
+ [scaled_height, scaled_width], InterpolationMode.BILINEAR, antialias=True
77
+ )(image)
78
+ image = torch.clip(image, 0.0, 1.0)
79
+ image = torch.permute(image, [1, 2, 0]).numpy()
80
+ else:
81
+ raise NotImplementedError(resize_method)
82
+
83
+ top_pad = (desired_height - scaled_height) // 2
84
+ left_pad = (desired_width - scaled_width) // 2
85
+ padding = [
86
+ [top_pad, desired_height - scaled_height - top_pad],
87
+ [left_pad, desired_width - scaled_width - left_pad],
88
+ [0, 0]
89
+ ]
90
+ image_mask = np.pad(np.ones_like(image[:, :, 0], dtype=bool), padding[:2])
91
+ image = np.pad(image, padding, constant_values=pad_value)
92
+ return image, image_mask
93
+
94
+
95
+ def select_tiling(h, w, patch_size, max_num_crops):
96
+ """Divide in image of size [w, h] in up to max_num_patches of size patch_size"""
97
+ original_size = np.stack([h, w]) # [1, 2]
98
+ original_res = h * w
99
+ tilings = []
100
+ for i in range(1, max_num_crops + 1):
101
+ for j in range(1, max_num_crops + 1):
102
+ if i*j <= max_num_crops:
103
+ tilings.append((i, j))
104
+ # sort so argmin and argmax favour smaller tilings in the event of a tie
105
+ tilings.sort(key=lambda x: (x[0]*x[1], x[0]))
106
+ candidate_tilings = np.array(tilings, dtype=np.int32) # [n_resolutions, 2]
107
+ candidate_resolutions = candidate_tilings * patch_size # [n_resolutions, 2]
108
+
109
+ # How much we would need to scale the image to fit exactly in each tiling
110
+ original_size = np.stack([h, w], dtype=np.float32) # [1, 2]
111
+ required_scale_d = candidate_resolutions.astype(np.float32) / original_size
112
+ required_scale = np.min(required_scale_d, axis=-1, keepdims=True) # [n_resolutions, 1]
113
+ if np.all(required_scale < 1):
114
+ # We are forced to downscale, so try to minimize the amount of downscaling
115
+ ix = np.argmax(required_scale)
116
+ else:
117
+ # Pick the resolution that required the least upscaling so that it most closely fits the image
118
+ required_scale = np.where(required_scale < 1.0, 10e9, required_scale)
119
+ ix = np.argmin(required_scale)
120
+ return candidate_tilings[ix]
121
+
122
+
123
+ def pixels_to_patches(array, patch_size):
124
+ """Reshape an image of [h, w, 3] -> [n_patches, pixels_per_patch]"""
125
+ w, h, c = array.shape
126
+ h_patches = h//patch_size
127
+ w_patches = w//patch_size
128
+ array = np.reshape(array, [h_patches, patch_size, w_patches, patch_size, c])
129
+ array = np.transpose(array, [0, 2, 1, 3, 4])
130
+ array = np.reshape(array, [h_patches*w_patches, patch_size*patch_size*c])
131
+ return array
132
+
133
+
134
+ def batch_pixels_to_patches(array, patch_size):
135
+ """Reshape images of [n_images, h, w, 3] -> [n_images, n_patches, pixels_per_patch]"""
136
+ if len(array.shape) == 3:
137
+ n_crops, w, h = array.shape
138
+ h_patches = h//patch_size
139
+ w_patches = w//patch_size
140
+ array = np.reshape(array, [n_crops, h_patches, patch_size, w_patches, patch_size])
141
+ array = np.transpose(array, [0, 1, 3, 2, 4])
142
+ array = np.reshape(array, [n_crops, h_patches*w_patches, patch_size*patch_size])
143
+ return array
144
+ else:
145
+ n_crops, w, h, c = array.shape
146
+ h_patches = h//patch_size
147
+ w_patches = w//patch_size
148
+ array = np.reshape(array, [n_crops, h_patches, patch_size, w_patches, patch_size, c])
149
+ array = np.transpose(array, [0, 1, 3, 2, 4, 5])
150
+ array = np.reshape(array, [n_crops, h_patches*w_patches, patch_size*patch_size*c])
151
+ return array
152
+
153
+
154
+ class MolmoImagesKwargs(ImagesKwargs, total=False):
155
+ max_crops: Optional[int]
156
+ overlap_margins: Optional[List[int]]
157
+ base_image_input_size: Optional[List[int]]
158
+ image_token_length_w: Optional[int]
159
+ image_token_length_h: Optional[int]
160
+ image_patch_size: Optional[int]
161
+ image_padding_mask: Optional[bool]
162
+
163
+
164
+ class MolmoImageProcessor(BaseImageProcessor):
165
+ """Preprocess images and multi-model inputs"""
166
+
167
+ def __init__(
168
+ self,
169
+ max_crops: int = 12,
170
+ overlap_margins: List[int] = (4, 4),
171
+ base_image_input_size: List[int] = (336, 336),
172
+ image_token_length_w: int = 12,
173
+ image_token_length_h: int = 12,
174
+ image_patch_size: int = 14,
175
+ image_padding_mask: bool = True,
176
+ do_normalize: bool = True,
177
+ **kwargs,
178
+ ):
179
+ super().__init__(**kwargs)
180
+ self.max_crops = max_crops
181
+ self.overlap_margins = overlap_margins
182
+ self.base_image_input_size = base_image_input_size
183
+ self.image_token_length_w = image_token_length_w
184
+ self.image_token_length_h = image_token_length_h
185
+ self.image_patch_size = image_patch_size
186
+ self.image_padding_mask = image_padding_mask
187
+ self.do_normalize = do_normalize
188
+
189
+ def _normalize(self, image):
190
+ if self.do_normalize:
191
+ image -= np.array(OPENAI_CLIP_MEAN, dtype=np.float32)[None, None, :]
192
+ image /= np.array(OPENAI_CLIP_STD, dtype=np.float32)[None, None, :]
193
+ return image
194
+
195
+ def image_to_patches_and_tokens(
196
+ self,
197
+ image: ImageInput,
198
+ image_patch_token_id: int,
199
+ image_col_token_id: int,
200
+ image_start_token_id: int,
201
+ image_end_token_id: int,
202
+ max_crops: Optional[int] = None,
203
+ overlap_margins: Optional[List[int]] = None,
204
+ base_image_input_size: Optional[Union[int, List[int]]] = None,
205
+ image_token_length_w: Optional[int] = None,
206
+ image_token_length_h: Optional[int] = None,
207
+ image_patch_size: Optional[int] = None,
208
+ ):
209
+ if isinstance(base_image_input_size, int):
210
+ base_image_input_size = (base_image_input_size, base_image_input_size)
211
+
212
+ base_image_input_d = image_patch_size
213
+ tokens_per_image = image_token_length_w * image_token_length_h
214
+ image_base_patch_w = base_image_input_size[1] // base_image_input_d
215
+ image_base_patch_h = base_image_input_size[0] // base_image_input_d
216
+
217
+ original_image_h, original_image_w = image.shape[:2]
218
+ crop_size = base_image_input_size[0]
219
+
220
+ # Discard this many patches from the (left/top, right/bottom) of crops
221
+ left_margin, right_margin = overlap_margins
222
+ # left_margin, right_margin = 2, 2
223
+ assert left_margin % 2 == 0 # Required for compatibility with 2x2 pooling
224
+ total_margin_pixels = base_image_input_d*(right_margin + left_margin) # pixels removed per dim
225
+ crop_patches = base_image_input_size[0] // base_image_input_d # patches per crop dim
226
+ crop_window_patches = crop_patches - (right_margin + left_margin) # usable patches
227
+ crop_window_size = crop_window_patches * base_image_input_d
228
+
229
+ # Decide how to tile the image, to account for the overlap margins we compute the tiling
230
+ # as if we had an image without the margins and were using a crop size without the margins
231
+ tiling = select_tiling(
232
+ original_image_h - total_margin_pixels,
233
+ original_image_w - total_margin_pixels,
234
+ crop_window_size,
235
+ max_crops
236
+ )
237
+ src, img_mask = resize_and_pad(
238
+ image,
239
+ [tiling[0]*crop_window_size+total_margin_pixels, tiling[1]*crop_window_size+total_margin_pixels]
240
+ )
241
+ src = self._normalize(src)
242
+
243
+ # Now we have to split the image into crops, while keeping track of how each patch in the
244
+ # each crop should be ordered in the global image, this require a lot of tricky booking
245
+ n_crops = tiling[0] * tiling[1]
246
+ patches_arr = []
247
+ mask_arr = []
248
+ patch_ordering_arr = []
249
+
250
+ # We assume 2x2 pooling, but can allow padding the right/bottom with extra
251
+ # patches if the number of patches per side is not even
252
+ assert (crop_patches+1)//2 == image_token_length_h
253
+ assert (crop_patches+1)//2 == image_token_length_w
254
+ on = 0
255
+ on_patch = 0
256
+ for i in range(tiling[0]):
257
+ y0 = i*crop_window_size
258
+ if i == 0:
259
+ crop_y0 = 0
260
+ else:
261
+ crop_y0 = left_margin // 2
262
+
263
+ crop_h = image_base_patch_h - (right_margin + left_margin)
264
+ if i == 0:
265
+ crop_h += left_margin
266
+ if i == (tiling[0]-1):
267
+ crop_h += right_margin
268
+ for j in range(tiling[1]):
269
+ x0 = j*crop_window_size
270
+ if j == 0:
271
+ crop_x0 = 0
272
+ else:
273
+ crop_x0 = left_margin // 2
274
+
275
+ crop_w = image_base_patch_w - (right_margin + left_margin)
276
+ if j == 0:
277
+ crop_w += left_margin
278
+ if j == (tiling[1]-1):
279
+ crop_w += right_margin
280
+
281
+ pooled_w = (crop_w + 1) // 2
282
+ pooled_h = (crop_h + 1) // 2
283
+ after_padding_width = image_token_length_w - pooled_w - crop_x0
284
+ after_padding_height = image_token_length_h - pooled_h - crop_y0
285
+ patch_ordering_arr.append(
286
+ np.pad(
287
+ np.reshape(
288
+ np.arange(on, on+pooled_h*pooled_w, dtype=np.int32),
289
+ (pooled_h, pooled_w)),
290
+ [[crop_y0, after_padding_height], [crop_x0, after_padding_width]],
291
+ constant_values=-1, mode='constant'
292
+ )
293
+ )
294
+ patches_arr.append(src[y0:y0+crop_size, x0:x0+crop_size])
295
+ mask_arr.append(img_mask[y0:y0+crop_size, x0:x0+crop_size])
296
+
297
+ on += pooled_h*pooled_w
298
+ on_patch += 1
299
+ patches = np.stack(patches_arr)
300
+ patch_ordering = np.stack(patch_ordering_arr)
301
+ img_mask = np.stack(mask_arr)
302
+
303
+ # Switch to [n_crops, n_patches, pixels_per_patch] format
304
+ image_layout_impatch_w, image_layout_impatch_h = tiling[0], tiling[1]
305
+
306
+ patches = batch_pixels_to_patches(patches, image_patch_size)
307
+ img_mask = batch_pixels_to_patches(img_mask, image_patch_size)
308
+ img_mask = img_mask.astype(np.float32).mean(axis=-1)
309
+ patch_ordering = np.reshape(patch_ordering, [-1])
310
+ valid = patch_ordering >= 0
311
+
312
+ # Path order numbers the patches crop-by-crop, here we transpose
313
+ # it to get left-to-right order
314
+ patch_ordering_rh = np.reshape(
315
+ patch_ordering,
316
+ [tiling[0], tiling[1], image_token_length_h, image_token_length_w]
317
+ )
318
+ patch_ordering_rh = np.transpose(patch_ordering_rh, [0, 2, 1, 3])
319
+ patch_ordering_rh = np.reshape(patch_ordering_rh, [-1])
320
+
321
+ # The transpose will screw up which patches are masked, project the
322
+ # new order into sparse structure of `patch_ordering` to fix it
323
+ patch_ordering[valid] = patch_ordering_rh[patch_ordering_rh >= 0]
324
+
325
+ # Now build the output tokens
326
+ h = tiling[0] * crop_window_patches + (right_margin+left_margin)
327
+ w = tiling[1] * crop_window_patches + (right_margin+left_margin)
328
+ per_row = np.full(
329
+ ((w+1)//2,),
330
+ image_patch_token_id,
331
+ )
332
+ per_row = np.concatenate([per_row, [image_col_token_id]], 0)
333
+
334
+ joint = np.tile(per_row, [(h+1)//2])
335
+ joint = [
336
+ [image_start_token_id],
337
+ joint,
338
+ [image_end_token_id]
339
+ ]
340
+
341
+ # Finally do the same for the global image
342
+ resized, _ = resize_and_pad(image, base_image_input_size)
343
+ resized = self._normalize(resized)
344
+ resized = pixels_to_patches(resized, image_patch_size)
345
+ patches = np.concatenate([np.expand_dims(resized, 0), patches], 0)
346
+
347
+ # Global image goes first, so the order of patches in previous crops gets increased
348
+ patch_ordering = np.where(
349
+ patch_ordering >= 0,
350
+ patch_ordering + tokens_per_image,
351
+ -1
352
+ )
353
+ patch_ordering = np.concatenate([np.arange(0, tokens_per_image), patch_ordering], 0)
354
+ per_row = np.full(
355
+ (image_token_length_w,),
356
+ image_patch_token_id,
357
+ )
358
+ per_row = np.concatenate([per_row, [image_col_token_id]], 0)
359
+ extra_tokens = np.tile(per_row, [image_token_length_h])
360
+ joint = [
361
+ [image_start_token_id],
362
+ extra_tokens,
363
+ [image_end_token_id],
364
+ ] + joint
365
+
366
+ joint = np.concatenate(joint, 0)
367
+ img_mask = np.pad(img_mask, [[0, 1], [0, 0]], constant_values=-1)
368
+ return patches, joint, patch_ordering, img_mask
369
+
370
+ def build_image_input_idx(
371
+ self,
372
+ image_tokens: np.ndarray,
373
+ patch_order: np.ndarray,
374
+ image_patch_token_id: int,
375
+ image_token_length_w: int,
376
+ image_token_length_h: int,
377
+ ):
378
+ """Converts `patch_order` into a mapping of token_id -> patch_id"""
379
+
380
+ tokens_per_image = image_token_length_w * image_token_length_h
381
+
382
+ # Indices to insert the patches
383
+ image_input_idx = image_tokens == image_patch_token_id
384
+ image_input_idx = np.nonzero(image_input_idx)[0].astype(np.int32)
385
+
386
+ if patch_order is not None:
387
+ n_tokens = image_input_idx.shape[0]
388
+ patch_order = np.reshape(patch_order, [-1])
389
+ n_patches = patch_order.shape[0]
390
+
391
+ valid = patch_order >= 0
392
+ n_valid_patches = valid.sum()
393
+ assert len(image_input_idx) == n_valid_patches
394
+
395
+ sorted_patch_ixs = np.zeros([n_tokens], np.int32)
396
+ sorted_patch_ixs[patch_order[valid]] = np.arange(n_valid_patches, dtype=np.int32)
397
+
398
+ # Project the inverted mapping into same sparse structure
399
+ sorted_patch_ixs_ex = np.full(np.shape(patch_order), -1)
400
+ sorted_patch_ixs_ex[valid] = sorted_patch_ixs
401
+
402
+ # Do the gather and then re-masked outputs that were masked in `sorted_patch_ixs`
403
+ valid = (sorted_patch_ixs_ex >= 0).astype(np.int32)
404
+ image_input_idx = image_input_idx[sorted_patch_ixs_ex*valid]
405
+ image_input_idx = image_input_idx*valid - 100*(1 - valid)
406
+ image_input_idx = np.reshape(image_input_idx, [-1, tokens_per_image])
407
+ return image_input_idx
408
+
409
+ def preprocess(
410
+ self,
411
+ image: np.ndarray,
412
+ image_patch_token_id: int,
413
+ image_col_token_id: int,
414
+ image_start_token_id: int,
415
+ image_end_token_id: int,
416
+ max_crops: Optional[int] = None,
417
+ overlap_margins: Optional[List[int]] = None,
418
+ base_image_input_size: Optional[Union[int, List[int]]] = None,
419
+ image_token_length_w: Optional[int] = None,
420
+ image_token_length_h: Optional[int] = None,
421
+ image_patch_size: Optional[int] = None,
422
+ **kwargs,
423
+ ):
424
+ """Preprocesses a single image
425
+
426
+ Returns:
427
+ crops: (n_crops, n_patches, patch_dim) individual crops, `n_crops` might
428
+ change between images but the other dimension are fixed
429
+ tokens: (n_tokens,) int32 tokens, pad tokens indicate where to insert the
430
+ patch features, might include other special tokens as well
431
+ image_idx: (n_crops, n_patches) index in `tokens` to put the patch features from the
432
+ crops after pooling, negative values indicates patches features to exclude
433
+ padding_mask: (n_crops, n_patches) what percent of each crop is padding, can be None
434
+ if the image mask is not being used.
435
+ """
436
+
437
+ max_crops = max_crops or self.max_crops
438
+ overlap_margins = overlap_margins or self.overlap_margins
439
+ base_image_input_size = base_image_input_size or self.base_image_input_size
440
+ image_token_length_w = image_token_length_w or self.image_token_length_w
441
+ image_token_length_h = image_token_length_h or self.image_token_length_h
442
+ image_patch_size = image_patch_size or self.image_patch_size
443
+
444
+ crops, image_tokens, patch_ordering, img_mask = self.image_to_patches_and_tokens(
445
+ image,
446
+ image_patch_token_id,
447
+ image_col_token_id,
448
+ image_start_token_id,
449
+ image_end_token_id,
450
+ max_crops,
451
+ overlap_margins,
452
+ base_image_input_size,
453
+ image_token_length_w,
454
+ image_token_length_h,
455
+ image_patch_size,
456
+ )
457
+ patch_idx = self.build_image_input_idx(
458
+ image_tokens,
459
+ patch_ordering,
460
+ image_patch_token_id,
461
+ image_token_length_w=image_token_length_w,
462
+ image_token_length_h=image_token_length_h,
463
+ )
464
+ return crops, image_tokens, patch_idx, img_mask
465
+
466
+ def multimodal_preprocess(
467
+ self,
468
+ images: np.ndarray,
469
+ tokens: List[int],
470
+ image_idx: np.ndarray,
471
+ sequence_length: int,
472
+ image_patch_token_id: int,
473
+ image_col_token_id: int,
474
+ image_start_token_id: int,
475
+ image_end_token_id: int,
476
+ **kwargs,
477
+ ):
478
+ """Merge images and text tokens into multi-modal features for the model
479
+
480
+ :param images: images to use as input
481
+ :param tokens: input text tokens
482
+ :param image_idx: where to insert the images into `tokens`
483
+ :params image_patch_token_id: id to use of tokens that will contain image features
484
+ :params image_col_token_id: token id for image column special tokens
485
+ :params image_start_token_id: token id for image start special tokens
486
+ :params image_end_token_id: token id for image end special tokens
487
+ :params kwargs: override preprocessor default args
488
+ """
489
+ if images is None:
490
+ return {"input_ids": tokens}
491
+
492
+ max_total_crops = kwargs.get("max_crops") or self.max_crops
493
+ image_token_length_w = kwargs.get("image_token_length_w") or self.image_token_length_w
494
+ image_token_length_h = kwargs.get("image_token_length_h") or self.image_token_length_h
495
+ image_patch_size = kwargs.get("image_patch_size") or self.image_patch_size
496
+ base_image_input_size = kwargs.get("base_image_input_size") or self.base_image_input_size
497
+ image_num_patch = (
498
+ base_image_input_size[0] // image_patch_size,
499
+ base_image_input_size[1] // image_patch_size,
500
+ )
501
+ image_padding_mask = kwargs.get("image_padding_mask") or self.image_padding_mask
502
+
503
+ tokens_per_image = image_token_length_w * image_token_length_h
504
+ n_pixels = image_patch_size * image_patch_size * 3
505
+ n_patches = image_num_patch[0] * image_num_patch[1]
506
+
507
+ n = len(images)
508
+ all_crops = []
509
+ all_image_idx = []
510
+ out_tokens = []
511
+ all_crop_masks = []
512
+
513
+ for ix in range(n):
514
+ token_ix = image_idx[ix]
515
+ crops, image_tokens, patch_idx, img_mask = self.preprocess(
516
+ images[ix],
517
+ image_patch_token_id,
518
+ image_col_token_id,
519
+ image_start_token_id,
520
+ image_end_token_id,
521
+ **kwargs,
522
+ )
523
+
524
+ if token_ix == -1: # -1 is an image inserted at the very start
525
+ start = 0
526
+ token_ix = 0
527
+ end = 0
528
+ else:
529
+ start = 0 if ix == 0 else image_idx[ix-1] + 1
530
+ end = token_ix + 1
531
+
532
+ all_image_idx.append(patch_idx + token_ix)
533
+ all_crops.append(crops)
534
+ out_tokens.append(tokens[start:token_ix])
535
+ out_tokens.append(image_tokens)
536
+ if ix == (n - 1):
537
+ out_tokens.append(tokens[end:])
538
+ if image_padding_mask:
539
+ all_crop_masks.append(img_mask)
540
+
541
+ input_ids = np.concatenate(out_tokens, 0)
542
+ images = np.concatenate(all_crops, 0)
543
+ image_input_idx = np.concatenate(all_image_idx, 0)
544
+ if image_padding_mask:
545
+ image_masks = np.concatenate(all_crop_masks, 0)
546
+ else:
547
+ image_masks = None
548
+
549
+ out = {
550
+ "input_ids": input_ids,
551
+ "images": images,
552
+ "image_input_idx": image_input_idx
553
+ }
554
+ if image_masks is not None:
555
+ out["image_masks"] = image_masks
556
+ return out
557
+
558
+
559
+ MolmoImageProcessor.register_for_auto_class()
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d66b3cef21535a5d4dd7cdd0cfe22aab76528f2f8a4c033ea13a6c4a5572afb7
3
+ size 4978535216
model-00002-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c497b343a4164a3909934bdfb90369d7532e01aafcbd6f4432251dd2fcba3d8
3
+ size 4778633832
model-00003-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:185e849c7de76fe344e042d723fbf698108ea192025051f30301b1fa2de32906
3
+ size 4661160096
model-00004-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d320ad1cdaa02cb9e41ca0896fedf29db1a2a7154824d7cfb304b167867a0aff
3
+ size 4661160112
model-00005-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f46f5768491c4bf572b05aba53f33c6d0bf25792e4af6b24ba3e0bb6d9acff8b
3
+ size 4661160112
model-00006-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bea1f2da30c1458e12088f39109b9ffe02bdb19c86b986748d88dffcb0aa90b7
3
+ size 4543686344
model-00007-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:279f80978caee24ecef2a8ac98c75cf06b488f5b9cf3b5701b120bcbcf9b8abb
3
+ size 3799841448
model.safetensors.index.json ADDED
@@ -0,0 +1,592 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 32084101120
4
+ },
5
+ "weight_map": {
6
+ "model.transformer.blocks.0.attn.att_proj.bias": "model-00001-of-00007.safetensors",
7
+ "model.transformer.blocks.0.attn.att_proj.weight": "model-00001-of-00007.safetensors",
8
+ "model.transformer.blocks.0.attn.attn_out.weight": "model-00001-of-00007.safetensors",
9
+ "model.transformer.blocks.0.attn_norm.weight": "model-00001-of-00007.safetensors",
10
+ "model.transformer.blocks.0.ff_norm.weight": "model-00001-of-00007.safetensors",
11
+ "model.transformer.blocks.0.mlp.ff_out.weight": "model-00001-of-00007.safetensors",
12
+ "model.transformer.blocks.0.mlp.ff_proj.weight": "model-00001-of-00007.safetensors",
13
+ "model.transformer.blocks.1.attn.att_proj.bias": "model-00001-of-00007.safetensors",
14
+ "model.transformer.blocks.1.attn.att_proj.weight": "model-00001-of-00007.safetensors",
15
+ "model.transformer.blocks.1.attn.attn_out.weight": "model-00001-of-00007.safetensors",
16
+ "model.transformer.blocks.1.attn_norm.weight": "model-00001-of-00007.safetensors",
17
+ "model.transformer.blocks.1.ff_norm.weight": "model-00001-of-00007.safetensors",
18
+ "model.transformer.blocks.1.mlp.ff_out.weight": "model-00001-of-00007.safetensors",
19
+ "model.transformer.blocks.1.mlp.ff_proj.weight": "model-00001-of-00007.safetensors",
20
+ "model.transformer.blocks.10.attn.att_proj.bias": "model-00003-of-00007.safetensors",
21
+ "model.transformer.blocks.10.attn.att_proj.weight": "model-00003-of-00007.safetensors",
22
+ "model.transformer.blocks.10.attn.attn_out.weight": "model-00003-of-00007.safetensors",
23
+ "model.transformer.blocks.10.attn_norm.weight": "model-00003-of-00007.safetensors",
24
+ "model.transformer.blocks.10.ff_norm.weight": "model-00003-of-00007.safetensors",
25
+ "model.transformer.blocks.10.mlp.ff_out.weight": "model-00003-of-00007.safetensors",
26
+ "model.transformer.blocks.10.mlp.ff_proj.weight": "model-00003-of-00007.safetensors",
27
+ "model.transformer.blocks.11.attn.att_proj.bias": "model-00003-of-00007.safetensors",
28
+ "model.transformer.blocks.11.attn.att_proj.weight": "model-00003-of-00007.safetensors",
29
+ "model.transformer.blocks.11.attn.attn_out.weight": "model-00003-of-00007.safetensors",
30
+ "model.transformer.blocks.11.attn_norm.weight": "model-00003-of-00007.safetensors",
31
+ "model.transformer.blocks.11.ff_norm.weight": "model-00003-of-00007.safetensors",
32
+ "model.transformer.blocks.11.mlp.ff_out.weight": "model-00003-of-00007.safetensors",
33
+ "model.transformer.blocks.11.mlp.ff_proj.weight": "model-00003-of-00007.safetensors",
34
+ "model.transformer.blocks.12.attn.att_proj.bias": "model-00003-of-00007.safetensors",
35
+ "model.transformer.blocks.12.attn.att_proj.weight": "model-00003-of-00007.safetensors",
36
+ "model.transformer.blocks.12.attn.attn_out.weight": "model-00003-of-00007.safetensors",
37
+ "model.transformer.blocks.12.attn_norm.weight": "model-00003-of-00007.safetensors",
38
+ "model.transformer.blocks.12.ff_norm.weight": "model-00003-of-00007.safetensors",
39
+ "model.transformer.blocks.12.mlp.ff_out.weight": "model-00003-of-00007.safetensors",
40
+ "model.transformer.blocks.12.mlp.ff_proj.weight": "model-00003-of-00007.safetensors",
41
+ "model.transformer.blocks.13.attn.att_proj.bias": "model-00003-of-00007.safetensors",
42
+ "model.transformer.blocks.13.attn.att_proj.weight": "model-00003-of-00007.safetensors",
43
+ "model.transformer.blocks.13.attn.attn_out.weight": "model-00003-of-00007.safetensors",
44
+ "model.transformer.blocks.13.attn_norm.weight": "model-00003-of-00007.safetensors",
45
+ "model.transformer.blocks.13.ff_norm.weight": "model-00004-of-00007.safetensors",
46
+ "model.transformer.blocks.13.mlp.ff_out.weight": "model-00004-of-00007.safetensors",
47
+ "model.transformer.blocks.13.mlp.ff_proj.weight": "model-00004-of-00007.safetensors",
48
+ "model.transformer.blocks.14.attn.att_proj.bias": "model-00004-of-00007.safetensors",
49
+ "model.transformer.blocks.14.attn.att_proj.weight": "model-00004-of-00007.safetensors",
50
+ "model.transformer.blocks.14.attn.attn_out.weight": "model-00004-of-00007.safetensors",
51
+ "model.transformer.blocks.14.attn_norm.weight": "model-00004-of-00007.safetensors",
52
+ "model.transformer.blocks.14.ff_norm.weight": "model-00004-of-00007.safetensors",
53
+ "model.transformer.blocks.14.mlp.ff_out.weight": "model-00004-of-00007.safetensors",
54
+ "model.transformer.blocks.14.mlp.ff_proj.weight": "model-00004-of-00007.safetensors",
55
+ "model.transformer.blocks.15.attn.att_proj.bias": "model-00004-of-00007.safetensors",
56
+ "model.transformer.blocks.15.attn.att_proj.weight": "model-00004-of-00007.safetensors",
57
+ "model.transformer.blocks.15.attn.attn_out.weight": "model-00004-of-00007.safetensors",
58
+ "model.transformer.blocks.15.attn_norm.weight": "model-00004-of-00007.safetensors",
59
+ "model.transformer.blocks.15.ff_norm.weight": "model-00004-of-00007.safetensors",
60
+ "model.transformer.blocks.15.mlp.ff_out.weight": "model-00004-of-00007.safetensors",
61
+ "model.transformer.blocks.15.mlp.ff_proj.weight": "model-00004-of-00007.safetensors",
62
+ "model.transformer.blocks.16.attn.att_proj.bias": "model-00004-of-00007.safetensors",
63
+ "model.transformer.blocks.16.attn.att_proj.weight": "model-00004-of-00007.safetensors",
64
+ "model.transformer.blocks.16.attn.attn_out.weight": "model-00004-of-00007.safetensors",
65
+ "model.transformer.blocks.16.attn_norm.weight": "model-00004-of-00007.safetensors",
66
+ "model.transformer.blocks.16.ff_norm.weight": "model-00004-of-00007.safetensors",
67
+ "model.transformer.blocks.16.mlp.ff_out.weight": "model-00004-of-00007.safetensors",
68
+ "model.transformer.blocks.16.mlp.ff_proj.weight": "model-00004-of-00007.safetensors",
69
+ "model.transformer.blocks.17.attn.att_proj.bias": "model-00004-of-00007.safetensors",
70
+ "model.transformer.blocks.17.attn.att_proj.weight": "model-00004-of-00007.safetensors",
71
+ "model.transformer.blocks.17.attn.attn_out.weight": "model-00004-of-00007.safetensors",
72
+ "model.transformer.blocks.17.attn_norm.weight": "model-00004-of-00007.safetensors",
73
+ "model.transformer.blocks.17.ff_norm.weight": "model-00004-of-00007.safetensors",
74
+ "model.transformer.blocks.17.mlp.ff_out.weight": "model-00004-of-00007.safetensors",
75
+ "model.transformer.blocks.17.mlp.ff_proj.weight": "model-00004-of-00007.safetensors",
76
+ "model.transformer.blocks.18.attn.att_proj.bias": "model-00004-of-00007.safetensors",
77
+ "model.transformer.blocks.18.attn.att_proj.weight": "model-00004-of-00007.safetensors",
78
+ "model.transformer.blocks.18.attn.attn_out.weight": "model-00004-of-00007.safetensors",
79
+ "model.transformer.blocks.18.attn_norm.weight": "model-00004-of-00007.safetensors",
80
+ "model.transformer.blocks.18.ff_norm.weight": "model-00005-of-00007.safetensors",
81
+ "model.transformer.blocks.18.mlp.ff_out.weight": "model-00005-of-00007.safetensors",
82
+ "model.transformer.blocks.18.mlp.ff_proj.weight": "model-00005-of-00007.safetensors",
83
+ "model.transformer.blocks.19.attn.att_proj.bias": "model-00005-of-00007.safetensors",
84
+ "model.transformer.blocks.19.attn.att_proj.weight": "model-00005-of-00007.safetensors",
85
+ "model.transformer.blocks.19.attn.attn_out.weight": "model-00005-of-00007.safetensors",
86
+ "model.transformer.blocks.19.attn_norm.weight": "model-00005-of-00007.safetensors",
87
+ "model.transformer.blocks.19.ff_norm.weight": "model-00005-of-00007.safetensors",
88
+ "model.transformer.blocks.19.mlp.ff_out.weight": "model-00005-of-00007.safetensors",
89
+ "model.transformer.blocks.19.mlp.ff_proj.weight": "model-00005-of-00007.safetensors",
90
+ "model.transformer.blocks.2.attn.att_proj.bias": "model-00001-of-00007.safetensors",
91
+ "model.transformer.blocks.2.attn.att_proj.weight": "model-00001-of-00007.safetensors",
92
+ "model.transformer.blocks.2.attn.attn_out.weight": "model-00001-of-00007.safetensors",
93
+ "model.transformer.blocks.2.attn_norm.weight": "model-00001-of-00007.safetensors",
94
+ "model.transformer.blocks.2.ff_norm.weight": "model-00001-of-00007.safetensors",
95
+ "model.transformer.blocks.2.mlp.ff_out.weight": "model-00001-of-00007.safetensors",
96
+ "model.transformer.blocks.2.mlp.ff_proj.weight": "model-00001-of-00007.safetensors",
97
+ "model.transformer.blocks.20.attn.att_proj.bias": "model-00005-of-00007.safetensors",
98
+ "model.transformer.blocks.20.attn.att_proj.weight": "model-00005-of-00007.safetensors",
99
+ "model.transformer.blocks.20.attn.attn_out.weight": "model-00005-of-00007.safetensors",
100
+ "model.transformer.blocks.20.attn_norm.weight": "model-00005-of-00007.safetensors",
101
+ "model.transformer.blocks.20.ff_norm.weight": "model-00005-of-00007.safetensors",
102
+ "model.transformer.blocks.20.mlp.ff_out.weight": "model-00005-of-00007.safetensors",
103
+ "model.transformer.blocks.20.mlp.ff_proj.weight": "model-00005-of-00007.safetensors",
104
+ "model.transformer.blocks.21.attn.att_proj.bias": "model-00005-of-00007.safetensors",
105
+ "model.transformer.blocks.21.attn.att_proj.weight": "model-00005-of-00007.safetensors",
106
+ "model.transformer.blocks.21.attn.attn_out.weight": "model-00005-of-00007.safetensors",
107
+ "model.transformer.blocks.21.attn_norm.weight": "model-00005-of-00007.safetensors",
108
+ "model.transformer.blocks.21.ff_norm.weight": "model-00005-of-00007.safetensors",
109
+ "model.transformer.blocks.21.mlp.ff_out.weight": "model-00005-of-00007.safetensors",
110
+ "model.transformer.blocks.21.mlp.ff_proj.weight": "model-00005-of-00007.safetensors",
111
+ "model.transformer.blocks.22.attn.att_proj.bias": "model-00005-of-00007.safetensors",
112
+ "model.transformer.blocks.22.attn.att_proj.weight": "model-00005-of-00007.safetensors",
113
+ "model.transformer.blocks.22.attn.attn_out.weight": "model-00005-of-00007.safetensors",
114
+ "model.transformer.blocks.22.attn_norm.weight": "model-00005-of-00007.safetensors",
115
+ "model.transformer.blocks.22.ff_norm.weight": "model-00005-of-00007.safetensors",
116
+ "model.transformer.blocks.22.mlp.ff_out.weight": "model-00005-of-00007.safetensors",
117
+ "model.transformer.blocks.22.mlp.ff_proj.weight": "model-00005-of-00007.safetensors",
118
+ "model.transformer.blocks.23.attn.att_proj.bias": "model-00005-of-00007.safetensors",
119
+ "model.transformer.blocks.23.attn.att_proj.weight": "model-00005-of-00007.safetensors",
120
+ "model.transformer.blocks.23.attn.attn_out.weight": "model-00005-of-00007.safetensors",
121
+ "model.transformer.blocks.23.attn_norm.weight": "model-00005-of-00007.safetensors",
122
+ "model.transformer.blocks.23.ff_norm.weight": "model-00006-of-00007.safetensors",
123
+ "model.transformer.blocks.23.mlp.ff_out.weight": "model-00006-of-00007.safetensors",
124
+ "model.transformer.blocks.23.mlp.ff_proj.weight": "model-00006-of-00007.safetensors",
125
+ "model.transformer.blocks.24.attn.att_proj.bias": "model-00006-of-00007.safetensors",
126
+ "model.transformer.blocks.24.attn.att_proj.weight": "model-00006-of-00007.safetensors",
127
+ "model.transformer.blocks.24.attn.attn_out.weight": "model-00006-of-00007.safetensors",
128
+ "model.transformer.blocks.24.attn_norm.weight": "model-00006-of-00007.safetensors",
129
+ "model.transformer.blocks.24.ff_norm.weight": "model-00006-of-00007.safetensors",
130
+ "model.transformer.blocks.24.mlp.ff_out.weight": "model-00006-of-00007.safetensors",
131
+ "model.transformer.blocks.24.mlp.ff_proj.weight": "model-00006-of-00007.safetensors",
132
+ "model.transformer.blocks.25.attn.att_proj.bias": "model-00006-of-00007.safetensors",
133
+ "model.transformer.blocks.25.attn.att_proj.weight": "model-00006-of-00007.safetensors",
134
+ "model.transformer.blocks.25.attn.attn_out.weight": "model-00006-of-00007.safetensors",
135
+ "model.transformer.blocks.25.attn_norm.weight": "model-00006-of-00007.safetensors",
136
+ "model.transformer.blocks.25.ff_norm.weight": "model-00006-of-00007.safetensors",
137
+ "model.transformer.blocks.25.mlp.ff_out.weight": "model-00006-of-00007.safetensors",
138
+ "model.transformer.blocks.25.mlp.ff_proj.weight": "model-00006-of-00007.safetensors",
139
+ "model.transformer.blocks.26.attn.att_proj.bias": "model-00006-of-00007.safetensors",
140
+ "model.transformer.blocks.26.attn.att_proj.weight": "model-00006-of-00007.safetensors",
141
+ "model.transformer.blocks.26.attn.attn_out.weight": "model-00006-of-00007.safetensors",
142
+ "model.transformer.blocks.26.attn_norm.weight": "model-00006-of-00007.safetensors",
143
+ "model.transformer.blocks.26.ff_norm.weight": "model-00006-of-00007.safetensors",
144
+ "model.transformer.blocks.26.mlp.ff_out.weight": "model-00006-of-00007.safetensors",
145
+ "model.transformer.blocks.26.mlp.ff_proj.weight": "model-00006-of-00007.safetensors",
146
+ "model.transformer.blocks.27.attn.att_proj.bias": "model-00006-of-00007.safetensors",
147
+ "model.transformer.blocks.27.attn.att_proj.weight": "model-00006-of-00007.safetensors",
148
+ "model.transformer.blocks.27.attn.attn_out.weight": "model-00006-of-00007.safetensors",
149
+ "model.transformer.blocks.27.attn_norm.weight": "model-00006-of-00007.safetensors",
150
+ "model.transformer.blocks.27.ff_norm.weight": "model-00006-of-00007.safetensors",
151
+ "model.transformer.blocks.27.mlp.ff_out.weight": "model-00006-of-00007.safetensors",
152
+ "model.transformer.blocks.27.mlp.ff_proj.weight": "model-00006-of-00007.safetensors",
153
+ "model.transformer.blocks.3.attn.att_proj.bias": "model-00002-of-00007.safetensors",
154
+ "model.transformer.blocks.3.attn.att_proj.weight": "model-00002-of-00007.safetensors",
155
+ "model.transformer.blocks.3.attn.attn_out.weight": "model-00002-of-00007.safetensors",
156
+ "model.transformer.blocks.3.attn_norm.weight": "model-00002-of-00007.safetensors",
157
+ "model.transformer.blocks.3.ff_norm.weight": "model-00002-of-00007.safetensors",
158
+ "model.transformer.blocks.3.mlp.ff_out.weight": "model-00002-of-00007.safetensors",
159
+ "model.transformer.blocks.3.mlp.ff_proj.weight": "model-00002-of-00007.safetensors",
160
+ "model.transformer.blocks.4.attn.att_proj.bias": "model-00002-of-00007.safetensors",
161
+ "model.transformer.blocks.4.attn.att_proj.weight": "model-00002-of-00007.safetensors",
162
+ "model.transformer.blocks.4.attn.attn_out.weight": "model-00002-of-00007.safetensors",
163
+ "model.transformer.blocks.4.attn_norm.weight": "model-00002-of-00007.safetensors",
164
+ "model.transformer.blocks.4.ff_norm.weight": "model-00002-of-00007.safetensors",
165
+ "model.transformer.blocks.4.mlp.ff_out.weight": "model-00002-of-00007.safetensors",
166
+ "model.transformer.blocks.4.mlp.ff_proj.weight": "model-00002-of-00007.safetensors",
167
+ "model.transformer.blocks.5.attn.att_proj.bias": "model-00002-of-00007.safetensors",
168
+ "model.transformer.blocks.5.attn.att_proj.weight": "model-00002-of-00007.safetensors",
169
+ "model.transformer.blocks.5.attn.attn_out.weight": "model-00002-of-00007.safetensors",
170
+ "model.transformer.blocks.5.attn_norm.weight": "model-00002-of-00007.safetensors",
171
+ "model.transformer.blocks.5.ff_norm.weight": "model-00002-of-00007.safetensors",
172
+ "model.transformer.blocks.5.mlp.ff_out.weight": "model-00002-of-00007.safetensors",
173
+ "model.transformer.blocks.5.mlp.ff_proj.weight": "model-00002-of-00007.safetensors",
174
+ "model.transformer.blocks.6.attn.att_proj.bias": "model-00002-of-00007.safetensors",
175
+ "model.transformer.blocks.6.attn.att_proj.weight": "model-00002-of-00007.safetensors",
176
+ "model.transformer.blocks.6.attn.attn_out.weight": "model-00002-of-00007.safetensors",
177
+ "model.transformer.blocks.6.attn_norm.weight": "model-00002-of-00007.safetensors",
178
+ "model.transformer.blocks.6.ff_norm.weight": "model-00002-of-00007.safetensors",
179
+ "model.transformer.blocks.6.mlp.ff_out.weight": "model-00002-of-00007.safetensors",
180
+ "model.transformer.blocks.6.mlp.ff_proj.weight": "model-00002-of-00007.safetensors",
181
+ "model.transformer.blocks.7.attn.att_proj.bias": "model-00002-of-00007.safetensors",
182
+ "model.transformer.blocks.7.attn.att_proj.weight": "model-00002-of-00007.safetensors",
183
+ "model.transformer.blocks.7.attn.attn_out.weight": "model-00002-of-00007.safetensors",
184
+ "model.transformer.blocks.7.attn_norm.weight": "model-00002-of-00007.safetensors",
185
+ "model.transformer.blocks.7.ff_norm.weight": "model-00002-of-00007.safetensors",
186
+ "model.transformer.blocks.7.mlp.ff_out.weight": "model-00002-of-00007.safetensors",
187
+ "model.transformer.blocks.7.mlp.ff_proj.weight": "model-00002-of-00007.safetensors",
188
+ "model.transformer.blocks.8.attn.att_proj.bias": "model-00002-of-00007.safetensors",
189
+ "model.transformer.blocks.8.attn.att_proj.weight": "model-00002-of-00007.safetensors",
190
+ "model.transformer.blocks.8.attn.attn_out.weight": "model-00002-of-00007.safetensors",
191
+ "model.transformer.blocks.8.attn_norm.weight": "model-00002-of-00007.safetensors",
192
+ "model.transformer.blocks.8.ff_norm.weight": "model-00003-of-00007.safetensors",
193
+ "model.transformer.blocks.8.mlp.ff_out.weight": "model-00003-of-00007.safetensors",
194
+ "model.transformer.blocks.8.mlp.ff_proj.weight": "model-00003-of-00007.safetensors",
195
+ "model.transformer.blocks.9.attn.att_proj.bias": "model-00003-of-00007.safetensors",
196
+ "model.transformer.blocks.9.attn.att_proj.weight": "model-00003-of-00007.safetensors",
197
+ "model.transformer.blocks.9.attn.attn_out.weight": "model-00003-of-00007.safetensors",
198
+ "model.transformer.blocks.9.attn_norm.weight": "model-00003-of-00007.safetensors",
199
+ "model.transformer.blocks.9.ff_norm.weight": "model-00003-of-00007.safetensors",
200
+ "model.transformer.blocks.9.mlp.ff_out.weight": "model-00003-of-00007.safetensors",
201
+ "model.transformer.blocks.9.mlp.ff_proj.weight": "model-00003-of-00007.safetensors",
202
+ "model.transformer.ff_out.weight": "model-00007-of-00007.safetensors",
203
+ "model.transformer.ln_f.weight": "model-00001-of-00007.safetensors",
204
+ "model.transformer.wte.embedding": "model-00001-of-00007.safetensors",
205
+ "model.transformer.wte.new_embedding": "model-00001-of-00007.safetensors",
206
+ "model.vision_backbone.image_pooling_2d.wk.bias": "model-00007-of-00007.safetensors",
207
+ "model.vision_backbone.image_pooling_2d.wk.weight": "model-00007-of-00007.safetensors",
208
+ "model.vision_backbone.image_pooling_2d.wo.bias": "model-00007-of-00007.safetensors",
209
+ "model.vision_backbone.image_pooling_2d.wo.weight": "model-00007-of-00007.safetensors",
210
+ "model.vision_backbone.image_pooling_2d.wq.bias": "model-00007-of-00007.safetensors",
211
+ "model.vision_backbone.image_pooling_2d.wq.weight": "model-00007-of-00007.safetensors",
212
+ "model.vision_backbone.image_pooling_2d.wv.bias": "model-00007-of-00007.safetensors",
213
+ "model.vision_backbone.image_pooling_2d.wv.weight": "model-00007-of-00007.safetensors",
214
+ "model.vision_backbone.image_projector.w1.weight": "model-00007-of-00007.safetensors",
215
+ "model.vision_backbone.image_projector.w2.weight": "model-00007-of-00007.safetensors",
216
+ "model.vision_backbone.image_projector.w3.weight": "model-00007-of-00007.safetensors",
217
+ "model.vision_backbone.image_vit.blocks.0.attention.wk.bias": "model-00007-of-00007.safetensors",
218
+ "model.vision_backbone.image_vit.blocks.0.attention.wk.weight": "model-00007-of-00007.safetensors",
219
+ "model.vision_backbone.image_vit.blocks.0.attention.wo.bias": "model-00007-of-00007.safetensors",
220
+ "model.vision_backbone.image_vit.blocks.0.attention.wo.weight": "model-00007-of-00007.safetensors",
221
+ "model.vision_backbone.image_vit.blocks.0.attention.wq.bias": "model-00007-of-00007.safetensors",
222
+ "model.vision_backbone.image_vit.blocks.0.attention.wq.weight": "model-00007-of-00007.safetensors",
223
+ "model.vision_backbone.image_vit.blocks.0.attention.wv.bias": "model-00007-of-00007.safetensors",
224
+ "model.vision_backbone.image_vit.blocks.0.attention.wv.weight": "model-00007-of-00007.safetensors",
225
+ "model.vision_backbone.image_vit.blocks.0.attention_norm.bias": "model-00007-of-00007.safetensors",
226
+ "model.vision_backbone.image_vit.blocks.0.attention_norm.weight": "model-00007-of-00007.safetensors",
227
+ "model.vision_backbone.image_vit.blocks.0.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
228
+ "model.vision_backbone.image_vit.blocks.0.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
229
+ "model.vision_backbone.image_vit.blocks.0.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
230
+ "model.vision_backbone.image_vit.blocks.0.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
231
+ "model.vision_backbone.image_vit.blocks.0.ffn_norm.bias": "model-00007-of-00007.safetensors",
232
+ "model.vision_backbone.image_vit.blocks.0.ffn_norm.weight": "model-00007-of-00007.safetensors",
233
+ "model.vision_backbone.image_vit.blocks.1.attention.wk.bias": "model-00007-of-00007.safetensors",
234
+ "model.vision_backbone.image_vit.blocks.1.attention.wk.weight": "model-00007-of-00007.safetensors",
235
+ "model.vision_backbone.image_vit.blocks.1.attention.wo.bias": "model-00007-of-00007.safetensors",
236
+ "model.vision_backbone.image_vit.blocks.1.attention.wo.weight": "model-00007-of-00007.safetensors",
237
+ "model.vision_backbone.image_vit.blocks.1.attention.wq.bias": "model-00007-of-00007.safetensors",
238
+ "model.vision_backbone.image_vit.blocks.1.attention.wq.weight": "model-00007-of-00007.safetensors",
239
+ "model.vision_backbone.image_vit.blocks.1.attention.wv.bias": "model-00007-of-00007.safetensors",
240
+ "model.vision_backbone.image_vit.blocks.1.attention.wv.weight": "model-00007-of-00007.safetensors",
241
+ "model.vision_backbone.image_vit.blocks.1.attention_norm.bias": "model-00007-of-00007.safetensors",
242
+ "model.vision_backbone.image_vit.blocks.1.attention_norm.weight": "model-00007-of-00007.safetensors",
243
+ "model.vision_backbone.image_vit.blocks.1.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
244
+ "model.vision_backbone.image_vit.blocks.1.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
245
+ "model.vision_backbone.image_vit.blocks.1.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
246
+ "model.vision_backbone.image_vit.blocks.1.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
247
+ "model.vision_backbone.image_vit.blocks.1.ffn_norm.bias": "model-00007-of-00007.safetensors",
248
+ "model.vision_backbone.image_vit.blocks.1.ffn_norm.weight": "model-00007-of-00007.safetensors",
249
+ "model.vision_backbone.image_vit.blocks.10.attention.wk.bias": "model-00007-of-00007.safetensors",
250
+ "model.vision_backbone.image_vit.blocks.10.attention.wk.weight": "model-00007-of-00007.safetensors",
251
+ "model.vision_backbone.image_vit.blocks.10.attention.wo.bias": "model-00007-of-00007.safetensors",
252
+ "model.vision_backbone.image_vit.blocks.10.attention.wo.weight": "model-00007-of-00007.safetensors",
253
+ "model.vision_backbone.image_vit.blocks.10.attention.wq.bias": "model-00007-of-00007.safetensors",
254
+ "model.vision_backbone.image_vit.blocks.10.attention.wq.weight": "model-00007-of-00007.safetensors",
255
+ "model.vision_backbone.image_vit.blocks.10.attention.wv.bias": "model-00007-of-00007.safetensors",
256
+ "model.vision_backbone.image_vit.blocks.10.attention.wv.weight": "model-00007-of-00007.safetensors",
257
+ "model.vision_backbone.image_vit.blocks.10.attention_norm.bias": "model-00007-of-00007.safetensors",
258
+ "model.vision_backbone.image_vit.blocks.10.attention_norm.weight": "model-00007-of-00007.safetensors",
259
+ "model.vision_backbone.image_vit.blocks.10.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
260
+ "model.vision_backbone.image_vit.blocks.10.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
261
+ "model.vision_backbone.image_vit.blocks.10.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
262
+ "model.vision_backbone.image_vit.blocks.10.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
263
+ "model.vision_backbone.image_vit.blocks.10.ffn_norm.bias": "model-00007-of-00007.safetensors",
264
+ "model.vision_backbone.image_vit.blocks.10.ffn_norm.weight": "model-00007-of-00007.safetensors",
265
+ "model.vision_backbone.image_vit.blocks.11.attention.wk.bias": "model-00007-of-00007.safetensors",
266
+ "model.vision_backbone.image_vit.blocks.11.attention.wk.weight": "model-00007-of-00007.safetensors",
267
+ "model.vision_backbone.image_vit.blocks.11.attention.wo.bias": "model-00007-of-00007.safetensors",
268
+ "model.vision_backbone.image_vit.blocks.11.attention.wo.weight": "model-00007-of-00007.safetensors",
269
+ "model.vision_backbone.image_vit.blocks.11.attention.wq.bias": "model-00007-of-00007.safetensors",
270
+ "model.vision_backbone.image_vit.blocks.11.attention.wq.weight": "model-00007-of-00007.safetensors",
271
+ "model.vision_backbone.image_vit.blocks.11.attention.wv.bias": "model-00007-of-00007.safetensors",
272
+ "model.vision_backbone.image_vit.blocks.11.attention.wv.weight": "model-00007-of-00007.safetensors",
273
+ "model.vision_backbone.image_vit.blocks.11.attention_norm.bias": "model-00007-of-00007.safetensors",
274
+ "model.vision_backbone.image_vit.blocks.11.attention_norm.weight": "model-00007-of-00007.safetensors",
275
+ "model.vision_backbone.image_vit.blocks.11.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
276
+ "model.vision_backbone.image_vit.blocks.11.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
277
+ "model.vision_backbone.image_vit.blocks.11.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
278
+ "model.vision_backbone.image_vit.blocks.11.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
279
+ "model.vision_backbone.image_vit.blocks.11.ffn_norm.bias": "model-00007-of-00007.safetensors",
280
+ "model.vision_backbone.image_vit.blocks.11.ffn_norm.weight": "model-00007-of-00007.safetensors",
281
+ "model.vision_backbone.image_vit.blocks.12.attention.wk.bias": "model-00007-of-00007.safetensors",
282
+ "model.vision_backbone.image_vit.blocks.12.attention.wk.weight": "model-00007-of-00007.safetensors",
283
+ "model.vision_backbone.image_vit.blocks.12.attention.wo.bias": "model-00007-of-00007.safetensors",
284
+ "model.vision_backbone.image_vit.blocks.12.attention.wo.weight": "model-00007-of-00007.safetensors",
285
+ "model.vision_backbone.image_vit.blocks.12.attention.wq.bias": "model-00007-of-00007.safetensors",
286
+ "model.vision_backbone.image_vit.blocks.12.attention.wq.weight": "model-00007-of-00007.safetensors",
287
+ "model.vision_backbone.image_vit.blocks.12.attention.wv.bias": "model-00007-of-00007.safetensors",
288
+ "model.vision_backbone.image_vit.blocks.12.attention.wv.weight": "model-00007-of-00007.safetensors",
289
+ "model.vision_backbone.image_vit.blocks.12.attention_norm.bias": "model-00007-of-00007.safetensors",
290
+ "model.vision_backbone.image_vit.blocks.12.attention_norm.weight": "model-00007-of-00007.safetensors",
291
+ "model.vision_backbone.image_vit.blocks.12.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
292
+ "model.vision_backbone.image_vit.blocks.12.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
293
+ "model.vision_backbone.image_vit.blocks.12.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
294
+ "model.vision_backbone.image_vit.blocks.12.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
295
+ "model.vision_backbone.image_vit.blocks.12.ffn_norm.bias": "model-00007-of-00007.safetensors",
296
+ "model.vision_backbone.image_vit.blocks.12.ffn_norm.weight": "model-00007-of-00007.safetensors",
297
+ "model.vision_backbone.image_vit.blocks.13.attention.wk.bias": "model-00007-of-00007.safetensors",
298
+ "model.vision_backbone.image_vit.blocks.13.attention.wk.weight": "model-00007-of-00007.safetensors",
299
+ "model.vision_backbone.image_vit.blocks.13.attention.wo.bias": "model-00007-of-00007.safetensors",
300
+ "model.vision_backbone.image_vit.blocks.13.attention.wo.weight": "model-00007-of-00007.safetensors",
301
+ "model.vision_backbone.image_vit.blocks.13.attention.wq.bias": "model-00007-of-00007.safetensors",
302
+ "model.vision_backbone.image_vit.blocks.13.attention.wq.weight": "model-00007-of-00007.safetensors",
303
+ "model.vision_backbone.image_vit.blocks.13.attention.wv.bias": "model-00007-of-00007.safetensors",
304
+ "model.vision_backbone.image_vit.blocks.13.attention.wv.weight": "model-00007-of-00007.safetensors",
305
+ "model.vision_backbone.image_vit.blocks.13.attention_norm.bias": "model-00007-of-00007.safetensors",
306
+ "model.vision_backbone.image_vit.blocks.13.attention_norm.weight": "model-00007-of-00007.safetensors",
307
+ "model.vision_backbone.image_vit.blocks.13.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
308
+ "model.vision_backbone.image_vit.blocks.13.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
309
+ "model.vision_backbone.image_vit.blocks.13.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
310
+ "model.vision_backbone.image_vit.blocks.13.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
311
+ "model.vision_backbone.image_vit.blocks.13.ffn_norm.bias": "model-00007-of-00007.safetensors",
312
+ "model.vision_backbone.image_vit.blocks.13.ffn_norm.weight": "model-00007-of-00007.safetensors",
313
+ "model.vision_backbone.image_vit.blocks.14.attention.wk.bias": "model-00007-of-00007.safetensors",
314
+ "model.vision_backbone.image_vit.blocks.14.attention.wk.weight": "model-00007-of-00007.safetensors",
315
+ "model.vision_backbone.image_vit.blocks.14.attention.wo.bias": "model-00007-of-00007.safetensors",
316
+ "model.vision_backbone.image_vit.blocks.14.attention.wo.weight": "model-00007-of-00007.safetensors",
317
+ "model.vision_backbone.image_vit.blocks.14.attention.wq.bias": "model-00007-of-00007.safetensors",
318
+ "model.vision_backbone.image_vit.blocks.14.attention.wq.weight": "model-00007-of-00007.safetensors",
319
+ "model.vision_backbone.image_vit.blocks.14.attention.wv.bias": "model-00007-of-00007.safetensors",
320
+ "model.vision_backbone.image_vit.blocks.14.attention.wv.weight": "model-00007-of-00007.safetensors",
321
+ "model.vision_backbone.image_vit.blocks.14.attention_norm.bias": "model-00007-of-00007.safetensors",
322
+ "model.vision_backbone.image_vit.blocks.14.attention_norm.weight": "model-00007-of-00007.safetensors",
323
+ "model.vision_backbone.image_vit.blocks.14.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
324
+ "model.vision_backbone.image_vit.blocks.14.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
325
+ "model.vision_backbone.image_vit.blocks.14.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
326
+ "model.vision_backbone.image_vit.blocks.14.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
327
+ "model.vision_backbone.image_vit.blocks.14.ffn_norm.bias": "model-00007-of-00007.safetensors",
328
+ "model.vision_backbone.image_vit.blocks.14.ffn_norm.weight": "model-00007-of-00007.safetensors",
329
+ "model.vision_backbone.image_vit.blocks.15.attention.wk.bias": "model-00007-of-00007.safetensors",
330
+ "model.vision_backbone.image_vit.blocks.15.attention.wk.weight": "model-00007-of-00007.safetensors",
331
+ "model.vision_backbone.image_vit.blocks.15.attention.wo.bias": "model-00007-of-00007.safetensors",
332
+ "model.vision_backbone.image_vit.blocks.15.attention.wo.weight": "model-00007-of-00007.safetensors",
333
+ "model.vision_backbone.image_vit.blocks.15.attention.wq.bias": "model-00007-of-00007.safetensors",
334
+ "model.vision_backbone.image_vit.blocks.15.attention.wq.weight": "model-00007-of-00007.safetensors",
335
+ "model.vision_backbone.image_vit.blocks.15.attention.wv.bias": "model-00007-of-00007.safetensors",
336
+ "model.vision_backbone.image_vit.blocks.15.attention.wv.weight": "model-00007-of-00007.safetensors",
337
+ "model.vision_backbone.image_vit.blocks.15.attention_norm.bias": "model-00007-of-00007.safetensors",
338
+ "model.vision_backbone.image_vit.blocks.15.attention_norm.weight": "model-00007-of-00007.safetensors",
339
+ "model.vision_backbone.image_vit.blocks.15.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
340
+ "model.vision_backbone.image_vit.blocks.15.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
341
+ "model.vision_backbone.image_vit.blocks.15.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
342
+ "model.vision_backbone.image_vit.blocks.15.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
343
+ "model.vision_backbone.image_vit.blocks.15.ffn_norm.bias": "model-00007-of-00007.safetensors",
344
+ "model.vision_backbone.image_vit.blocks.15.ffn_norm.weight": "model-00007-of-00007.safetensors",
345
+ "model.vision_backbone.image_vit.blocks.16.attention.wk.bias": "model-00007-of-00007.safetensors",
346
+ "model.vision_backbone.image_vit.blocks.16.attention.wk.weight": "model-00007-of-00007.safetensors",
347
+ "model.vision_backbone.image_vit.blocks.16.attention.wo.bias": "model-00007-of-00007.safetensors",
348
+ "model.vision_backbone.image_vit.blocks.16.attention.wo.weight": "model-00007-of-00007.safetensors",
349
+ "model.vision_backbone.image_vit.blocks.16.attention.wq.bias": "model-00007-of-00007.safetensors",
350
+ "model.vision_backbone.image_vit.blocks.16.attention.wq.weight": "model-00007-of-00007.safetensors",
351
+ "model.vision_backbone.image_vit.blocks.16.attention.wv.bias": "model-00007-of-00007.safetensors",
352
+ "model.vision_backbone.image_vit.blocks.16.attention.wv.weight": "model-00007-of-00007.safetensors",
353
+ "model.vision_backbone.image_vit.blocks.16.attention_norm.bias": "model-00007-of-00007.safetensors",
354
+ "model.vision_backbone.image_vit.blocks.16.attention_norm.weight": "model-00007-of-00007.safetensors",
355
+ "model.vision_backbone.image_vit.blocks.16.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
356
+ "model.vision_backbone.image_vit.blocks.16.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
357
+ "model.vision_backbone.image_vit.blocks.16.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
358
+ "model.vision_backbone.image_vit.blocks.16.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
359
+ "model.vision_backbone.image_vit.blocks.16.ffn_norm.bias": "model-00007-of-00007.safetensors",
360
+ "model.vision_backbone.image_vit.blocks.16.ffn_norm.weight": "model-00007-of-00007.safetensors",
361
+ "model.vision_backbone.image_vit.blocks.17.attention.wk.bias": "model-00007-of-00007.safetensors",
362
+ "model.vision_backbone.image_vit.blocks.17.attention.wk.weight": "model-00007-of-00007.safetensors",
363
+ "model.vision_backbone.image_vit.blocks.17.attention.wo.bias": "model-00007-of-00007.safetensors",
364
+ "model.vision_backbone.image_vit.blocks.17.attention.wo.weight": "model-00007-of-00007.safetensors",
365
+ "model.vision_backbone.image_vit.blocks.17.attention.wq.bias": "model-00007-of-00007.safetensors",
366
+ "model.vision_backbone.image_vit.blocks.17.attention.wq.weight": "model-00007-of-00007.safetensors",
367
+ "model.vision_backbone.image_vit.blocks.17.attention.wv.bias": "model-00007-of-00007.safetensors",
368
+ "model.vision_backbone.image_vit.blocks.17.attention.wv.weight": "model-00007-of-00007.safetensors",
369
+ "model.vision_backbone.image_vit.blocks.17.attention_norm.bias": "model-00007-of-00007.safetensors",
370
+ "model.vision_backbone.image_vit.blocks.17.attention_norm.weight": "model-00007-of-00007.safetensors",
371
+ "model.vision_backbone.image_vit.blocks.17.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
372
+ "model.vision_backbone.image_vit.blocks.17.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
373
+ "model.vision_backbone.image_vit.blocks.17.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
374
+ "model.vision_backbone.image_vit.blocks.17.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
375
+ "model.vision_backbone.image_vit.blocks.17.ffn_norm.bias": "model-00007-of-00007.safetensors",
376
+ "model.vision_backbone.image_vit.blocks.17.ffn_norm.weight": "model-00007-of-00007.safetensors",
377
+ "model.vision_backbone.image_vit.blocks.18.attention.wk.bias": "model-00007-of-00007.safetensors",
378
+ "model.vision_backbone.image_vit.blocks.18.attention.wk.weight": "model-00007-of-00007.safetensors",
379
+ "model.vision_backbone.image_vit.blocks.18.attention.wo.bias": "model-00007-of-00007.safetensors",
380
+ "model.vision_backbone.image_vit.blocks.18.attention.wo.weight": "model-00007-of-00007.safetensors",
381
+ "model.vision_backbone.image_vit.blocks.18.attention.wq.bias": "model-00007-of-00007.safetensors",
382
+ "model.vision_backbone.image_vit.blocks.18.attention.wq.weight": "model-00007-of-00007.safetensors",
383
+ "model.vision_backbone.image_vit.blocks.18.attention.wv.bias": "model-00007-of-00007.safetensors",
384
+ "model.vision_backbone.image_vit.blocks.18.attention.wv.weight": "model-00007-of-00007.safetensors",
385
+ "model.vision_backbone.image_vit.blocks.18.attention_norm.bias": "model-00007-of-00007.safetensors",
386
+ "model.vision_backbone.image_vit.blocks.18.attention_norm.weight": "model-00007-of-00007.safetensors",
387
+ "model.vision_backbone.image_vit.blocks.18.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
388
+ "model.vision_backbone.image_vit.blocks.18.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
389
+ "model.vision_backbone.image_vit.blocks.18.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
390
+ "model.vision_backbone.image_vit.blocks.18.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
391
+ "model.vision_backbone.image_vit.blocks.18.ffn_norm.bias": "model-00007-of-00007.safetensors",
392
+ "model.vision_backbone.image_vit.blocks.18.ffn_norm.weight": "model-00007-of-00007.safetensors",
393
+ "model.vision_backbone.image_vit.blocks.19.attention.wk.bias": "model-00007-of-00007.safetensors",
394
+ "model.vision_backbone.image_vit.blocks.19.attention.wk.weight": "model-00007-of-00007.safetensors",
395
+ "model.vision_backbone.image_vit.blocks.19.attention.wo.bias": "model-00007-of-00007.safetensors",
396
+ "model.vision_backbone.image_vit.blocks.19.attention.wo.weight": "model-00007-of-00007.safetensors",
397
+ "model.vision_backbone.image_vit.blocks.19.attention.wq.bias": "model-00007-of-00007.safetensors",
398
+ "model.vision_backbone.image_vit.blocks.19.attention.wq.weight": "model-00007-of-00007.safetensors",
399
+ "model.vision_backbone.image_vit.blocks.19.attention.wv.bias": "model-00007-of-00007.safetensors",
400
+ "model.vision_backbone.image_vit.blocks.19.attention.wv.weight": "model-00007-of-00007.safetensors",
401
+ "model.vision_backbone.image_vit.blocks.19.attention_norm.bias": "model-00007-of-00007.safetensors",
402
+ "model.vision_backbone.image_vit.blocks.19.attention_norm.weight": "model-00007-of-00007.safetensors",
403
+ "model.vision_backbone.image_vit.blocks.19.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
404
+ "model.vision_backbone.image_vit.blocks.19.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
405
+ "model.vision_backbone.image_vit.blocks.19.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
406
+ "model.vision_backbone.image_vit.blocks.19.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
407
+ "model.vision_backbone.image_vit.blocks.19.ffn_norm.bias": "model-00007-of-00007.safetensors",
408
+ "model.vision_backbone.image_vit.blocks.19.ffn_norm.weight": "model-00007-of-00007.safetensors",
409
+ "model.vision_backbone.image_vit.blocks.2.attention.wk.bias": "model-00007-of-00007.safetensors",
410
+ "model.vision_backbone.image_vit.blocks.2.attention.wk.weight": "model-00007-of-00007.safetensors",
411
+ "model.vision_backbone.image_vit.blocks.2.attention.wo.bias": "model-00007-of-00007.safetensors",
412
+ "model.vision_backbone.image_vit.blocks.2.attention.wo.weight": "model-00007-of-00007.safetensors",
413
+ "model.vision_backbone.image_vit.blocks.2.attention.wq.bias": "model-00007-of-00007.safetensors",
414
+ "model.vision_backbone.image_vit.blocks.2.attention.wq.weight": "model-00007-of-00007.safetensors",
415
+ "model.vision_backbone.image_vit.blocks.2.attention.wv.bias": "model-00007-of-00007.safetensors",
416
+ "model.vision_backbone.image_vit.blocks.2.attention.wv.weight": "model-00007-of-00007.safetensors",
417
+ "model.vision_backbone.image_vit.blocks.2.attention_norm.bias": "model-00007-of-00007.safetensors",
418
+ "model.vision_backbone.image_vit.blocks.2.attention_norm.weight": "model-00007-of-00007.safetensors",
419
+ "model.vision_backbone.image_vit.blocks.2.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
420
+ "model.vision_backbone.image_vit.blocks.2.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
421
+ "model.vision_backbone.image_vit.blocks.2.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
422
+ "model.vision_backbone.image_vit.blocks.2.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
423
+ "model.vision_backbone.image_vit.blocks.2.ffn_norm.bias": "model-00007-of-00007.safetensors",
424
+ "model.vision_backbone.image_vit.blocks.2.ffn_norm.weight": "model-00007-of-00007.safetensors",
425
+ "model.vision_backbone.image_vit.blocks.20.attention.wk.bias": "model-00007-of-00007.safetensors",
426
+ "model.vision_backbone.image_vit.blocks.20.attention.wk.weight": "model-00007-of-00007.safetensors",
427
+ "model.vision_backbone.image_vit.blocks.20.attention.wo.bias": "model-00007-of-00007.safetensors",
428
+ "model.vision_backbone.image_vit.blocks.20.attention.wo.weight": "model-00007-of-00007.safetensors",
429
+ "model.vision_backbone.image_vit.blocks.20.attention.wq.bias": "model-00007-of-00007.safetensors",
430
+ "model.vision_backbone.image_vit.blocks.20.attention.wq.weight": "model-00007-of-00007.safetensors",
431
+ "model.vision_backbone.image_vit.blocks.20.attention.wv.bias": "model-00007-of-00007.safetensors",
432
+ "model.vision_backbone.image_vit.blocks.20.attention.wv.weight": "model-00007-of-00007.safetensors",
433
+ "model.vision_backbone.image_vit.blocks.20.attention_norm.bias": "model-00007-of-00007.safetensors",
434
+ "model.vision_backbone.image_vit.blocks.20.attention_norm.weight": "model-00007-of-00007.safetensors",
435
+ "model.vision_backbone.image_vit.blocks.20.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
436
+ "model.vision_backbone.image_vit.blocks.20.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
437
+ "model.vision_backbone.image_vit.blocks.20.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
438
+ "model.vision_backbone.image_vit.blocks.20.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
439
+ "model.vision_backbone.image_vit.blocks.20.ffn_norm.bias": "model-00007-of-00007.safetensors",
440
+ "model.vision_backbone.image_vit.blocks.20.ffn_norm.weight": "model-00007-of-00007.safetensors",
441
+ "model.vision_backbone.image_vit.blocks.21.attention.wk.bias": "model-00007-of-00007.safetensors",
442
+ "model.vision_backbone.image_vit.blocks.21.attention.wk.weight": "model-00007-of-00007.safetensors",
443
+ "model.vision_backbone.image_vit.blocks.21.attention.wo.bias": "model-00007-of-00007.safetensors",
444
+ "model.vision_backbone.image_vit.blocks.21.attention.wo.weight": "model-00007-of-00007.safetensors",
445
+ "model.vision_backbone.image_vit.blocks.21.attention.wq.bias": "model-00007-of-00007.safetensors",
446
+ "model.vision_backbone.image_vit.blocks.21.attention.wq.weight": "model-00007-of-00007.safetensors",
447
+ "model.vision_backbone.image_vit.blocks.21.attention.wv.bias": "model-00007-of-00007.safetensors",
448
+ "model.vision_backbone.image_vit.blocks.21.attention.wv.weight": "model-00007-of-00007.safetensors",
449
+ "model.vision_backbone.image_vit.blocks.21.attention_norm.bias": "model-00007-of-00007.safetensors",
450
+ "model.vision_backbone.image_vit.blocks.21.attention_norm.weight": "model-00007-of-00007.safetensors",
451
+ "model.vision_backbone.image_vit.blocks.21.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
452
+ "model.vision_backbone.image_vit.blocks.21.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
453
+ "model.vision_backbone.image_vit.blocks.21.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
454
+ "model.vision_backbone.image_vit.blocks.21.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
455
+ "model.vision_backbone.image_vit.blocks.21.ffn_norm.bias": "model-00007-of-00007.safetensors",
456
+ "model.vision_backbone.image_vit.blocks.21.ffn_norm.weight": "model-00007-of-00007.safetensors",
457
+ "model.vision_backbone.image_vit.blocks.22.attention.wk.bias": "model-00007-of-00007.safetensors",
458
+ "model.vision_backbone.image_vit.blocks.22.attention.wk.weight": "model-00007-of-00007.safetensors",
459
+ "model.vision_backbone.image_vit.blocks.22.attention.wo.bias": "model-00007-of-00007.safetensors",
460
+ "model.vision_backbone.image_vit.blocks.22.attention.wo.weight": "model-00007-of-00007.safetensors",
461
+ "model.vision_backbone.image_vit.blocks.22.attention.wq.bias": "model-00007-of-00007.safetensors",
462
+ "model.vision_backbone.image_vit.blocks.22.attention.wq.weight": "model-00007-of-00007.safetensors",
463
+ "model.vision_backbone.image_vit.blocks.22.attention.wv.bias": "model-00007-of-00007.safetensors",
464
+ "model.vision_backbone.image_vit.blocks.22.attention.wv.weight": "model-00007-of-00007.safetensors",
465
+ "model.vision_backbone.image_vit.blocks.22.attention_norm.bias": "model-00007-of-00007.safetensors",
466
+ "model.vision_backbone.image_vit.blocks.22.attention_norm.weight": "model-00007-of-00007.safetensors",
467
+ "model.vision_backbone.image_vit.blocks.22.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
468
+ "model.vision_backbone.image_vit.blocks.22.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
469
+ "model.vision_backbone.image_vit.blocks.22.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
470
+ "model.vision_backbone.image_vit.blocks.22.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
471
+ "model.vision_backbone.image_vit.blocks.22.ffn_norm.bias": "model-00007-of-00007.safetensors",
472
+ "model.vision_backbone.image_vit.blocks.22.ffn_norm.weight": "model-00007-of-00007.safetensors",
473
+ "model.vision_backbone.image_vit.blocks.3.attention.wk.bias": "model-00007-of-00007.safetensors",
474
+ "model.vision_backbone.image_vit.blocks.3.attention.wk.weight": "model-00007-of-00007.safetensors",
475
+ "model.vision_backbone.image_vit.blocks.3.attention.wo.bias": "model-00007-of-00007.safetensors",
476
+ "model.vision_backbone.image_vit.blocks.3.attention.wo.weight": "model-00007-of-00007.safetensors",
477
+ "model.vision_backbone.image_vit.blocks.3.attention.wq.bias": "model-00007-of-00007.safetensors",
478
+ "model.vision_backbone.image_vit.blocks.3.attention.wq.weight": "model-00007-of-00007.safetensors",
479
+ "model.vision_backbone.image_vit.blocks.3.attention.wv.bias": "model-00007-of-00007.safetensors",
480
+ "model.vision_backbone.image_vit.blocks.3.attention.wv.weight": "model-00007-of-00007.safetensors",
481
+ "model.vision_backbone.image_vit.blocks.3.attention_norm.bias": "model-00007-of-00007.safetensors",
482
+ "model.vision_backbone.image_vit.blocks.3.attention_norm.weight": "model-00007-of-00007.safetensors",
483
+ "model.vision_backbone.image_vit.blocks.3.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
484
+ "model.vision_backbone.image_vit.blocks.3.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
485
+ "model.vision_backbone.image_vit.blocks.3.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
486
+ "model.vision_backbone.image_vit.blocks.3.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
487
+ "model.vision_backbone.image_vit.blocks.3.ffn_norm.bias": "model-00007-of-00007.safetensors",
488
+ "model.vision_backbone.image_vit.blocks.3.ffn_norm.weight": "model-00007-of-00007.safetensors",
489
+ "model.vision_backbone.image_vit.blocks.4.attention.wk.bias": "model-00007-of-00007.safetensors",
490
+ "model.vision_backbone.image_vit.blocks.4.attention.wk.weight": "model-00007-of-00007.safetensors",
491
+ "model.vision_backbone.image_vit.blocks.4.attention.wo.bias": "model-00007-of-00007.safetensors",
492
+ "model.vision_backbone.image_vit.blocks.4.attention.wo.weight": "model-00007-of-00007.safetensors",
493
+ "model.vision_backbone.image_vit.blocks.4.attention.wq.bias": "model-00007-of-00007.safetensors",
494
+ "model.vision_backbone.image_vit.blocks.4.attention.wq.weight": "model-00007-of-00007.safetensors",
495
+ "model.vision_backbone.image_vit.blocks.4.attention.wv.bias": "model-00007-of-00007.safetensors",
496
+ "model.vision_backbone.image_vit.blocks.4.attention.wv.weight": "model-00007-of-00007.safetensors",
497
+ "model.vision_backbone.image_vit.blocks.4.attention_norm.bias": "model-00007-of-00007.safetensors",
498
+ "model.vision_backbone.image_vit.blocks.4.attention_norm.weight": "model-00007-of-00007.safetensors",
499
+ "model.vision_backbone.image_vit.blocks.4.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
500
+ "model.vision_backbone.image_vit.blocks.4.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
501
+ "model.vision_backbone.image_vit.blocks.4.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
502
+ "model.vision_backbone.image_vit.blocks.4.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
503
+ "model.vision_backbone.image_vit.blocks.4.ffn_norm.bias": "model-00007-of-00007.safetensors",
504
+ "model.vision_backbone.image_vit.blocks.4.ffn_norm.weight": "model-00007-of-00007.safetensors",
505
+ "model.vision_backbone.image_vit.blocks.5.attention.wk.bias": "model-00007-of-00007.safetensors",
506
+ "model.vision_backbone.image_vit.blocks.5.attention.wk.weight": "model-00007-of-00007.safetensors",
507
+ "model.vision_backbone.image_vit.blocks.5.attention.wo.bias": "model-00007-of-00007.safetensors",
508
+ "model.vision_backbone.image_vit.blocks.5.attention.wo.weight": "model-00007-of-00007.safetensors",
509
+ "model.vision_backbone.image_vit.blocks.5.attention.wq.bias": "model-00007-of-00007.safetensors",
510
+ "model.vision_backbone.image_vit.blocks.5.attention.wq.weight": "model-00007-of-00007.safetensors",
511
+ "model.vision_backbone.image_vit.blocks.5.attention.wv.bias": "model-00007-of-00007.safetensors",
512
+ "model.vision_backbone.image_vit.blocks.5.attention.wv.weight": "model-00007-of-00007.safetensors",
513
+ "model.vision_backbone.image_vit.blocks.5.attention_norm.bias": "model-00007-of-00007.safetensors",
514
+ "model.vision_backbone.image_vit.blocks.5.attention_norm.weight": "model-00007-of-00007.safetensors",
515
+ "model.vision_backbone.image_vit.blocks.5.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
516
+ "model.vision_backbone.image_vit.blocks.5.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
517
+ "model.vision_backbone.image_vit.blocks.5.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
518
+ "model.vision_backbone.image_vit.blocks.5.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
519
+ "model.vision_backbone.image_vit.blocks.5.ffn_norm.bias": "model-00007-of-00007.safetensors",
520
+ "model.vision_backbone.image_vit.blocks.5.ffn_norm.weight": "model-00007-of-00007.safetensors",
521
+ "model.vision_backbone.image_vit.blocks.6.attention.wk.bias": "model-00007-of-00007.safetensors",
522
+ "model.vision_backbone.image_vit.blocks.6.attention.wk.weight": "model-00007-of-00007.safetensors",
523
+ "model.vision_backbone.image_vit.blocks.6.attention.wo.bias": "model-00007-of-00007.safetensors",
524
+ "model.vision_backbone.image_vit.blocks.6.attention.wo.weight": "model-00007-of-00007.safetensors",
525
+ "model.vision_backbone.image_vit.blocks.6.attention.wq.bias": "model-00007-of-00007.safetensors",
526
+ "model.vision_backbone.image_vit.blocks.6.attention.wq.weight": "model-00007-of-00007.safetensors",
527
+ "model.vision_backbone.image_vit.blocks.6.attention.wv.bias": "model-00007-of-00007.safetensors",
528
+ "model.vision_backbone.image_vit.blocks.6.attention.wv.weight": "model-00007-of-00007.safetensors",
529
+ "model.vision_backbone.image_vit.blocks.6.attention_norm.bias": "model-00007-of-00007.safetensors",
530
+ "model.vision_backbone.image_vit.blocks.6.attention_norm.weight": "model-00007-of-00007.safetensors",
531
+ "model.vision_backbone.image_vit.blocks.6.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
532
+ "model.vision_backbone.image_vit.blocks.6.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
533
+ "model.vision_backbone.image_vit.blocks.6.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
534
+ "model.vision_backbone.image_vit.blocks.6.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
535
+ "model.vision_backbone.image_vit.blocks.6.ffn_norm.bias": "model-00007-of-00007.safetensors",
536
+ "model.vision_backbone.image_vit.blocks.6.ffn_norm.weight": "model-00007-of-00007.safetensors",
537
+ "model.vision_backbone.image_vit.blocks.7.attention.wk.bias": "model-00007-of-00007.safetensors",
538
+ "model.vision_backbone.image_vit.blocks.7.attention.wk.weight": "model-00007-of-00007.safetensors",
539
+ "model.vision_backbone.image_vit.blocks.7.attention.wo.bias": "model-00007-of-00007.safetensors",
540
+ "model.vision_backbone.image_vit.blocks.7.attention.wo.weight": "model-00007-of-00007.safetensors",
541
+ "model.vision_backbone.image_vit.blocks.7.attention.wq.bias": "model-00007-of-00007.safetensors",
542
+ "model.vision_backbone.image_vit.blocks.7.attention.wq.weight": "model-00007-of-00007.safetensors",
543
+ "model.vision_backbone.image_vit.blocks.7.attention.wv.bias": "model-00007-of-00007.safetensors",
544
+ "model.vision_backbone.image_vit.blocks.7.attention.wv.weight": "model-00007-of-00007.safetensors",
545
+ "model.vision_backbone.image_vit.blocks.7.attention_norm.bias": "model-00007-of-00007.safetensors",
546
+ "model.vision_backbone.image_vit.blocks.7.attention_norm.weight": "model-00007-of-00007.safetensors",
547
+ "model.vision_backbone.image_vit.blocks.7.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
548
+ "model.vision_backbone.image_vit.blocks.7.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
549
+ "model.vision_backbone.image_vit.blocks.7.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
550
+ "model.vision_backbone.image_vit.blocks.7.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
551
+ "model.vision_backbone.image_vit.blocks.7.ffn_norm.bias": "model-00007-of-00007.safetensors",
552
+ "model.vision_backbone.image_vit.blocks.7.ffn_norm.weight": "model-00007-of-00007.safetensors",
553
+ "model.vision_backbone.image_vit.blocks.8.attention.wk.bias": "model-00007-of-00007.safetensors",
554
+ "model.vision_backbone.image_vit.blocks.8.attention.wk.weight": "model-00007-of-00007.safetensors",
555
+ "model.vision_backbone.image_vit.blocks.8.attention.wo.bias": "model-00007-of-00007.safetensors",
556
+ "model.vision_backbone.image_vit.blocks.8.attention.wo.weight": "model-00007-of-00007.safetensors",
557
+ "model.vision_backbone.image_vit.blocks.8.attention.wq.bias": "model-00007-of-00007.safetensors",
558
+ "model.vision_backbone.image_vit.blocks.8.attention.wq.weight": "model-00007-of-00007.safetensors",
559
+ "model.vision_backbone.image_vit.blocks.8.attention.wv.bias": "model-00007-of-00007.safetensors",
560
+ "model.vision_backbone.image_vit.blocks.8.attention.wv.weight": "model-00007-of-00007.safetensors",
561
+ "model.vision_backbone.image_vit.blocks.8.attention_norm.bias": "model-00007-of-00007.safetensors",
562
+ "model.vision_backbone.image_vit.blocks.8.attention_norm.weight": "model-00007-of-00007.safetensors",
563
+ "model.vision_backbone.image_vit.blocks.8.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
564
+ "model.vision_backbone.image_vit.blocks.8.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
565
+ "model.vision_backbone.image_vit.blocks.8.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
566
+ "model.vision_backbone.image_vit.blocks.8.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
567
+ "model.vision_backbone.image_vit.blocks.8.ffn_norm.bias": "model-00007-of-00007.safetensors",
568
+ "model.vision_backbone.image_vit.blocks.8.ffn_norm.weight": "model-00007-of-00007.safetensors",
569
+ "model.vision_backbone.image_vit.blocks.9.attention.wk.bias": "model-00007-of-00007.safetensors",
570
+ "model.vision_backbone.image_vit.blocks.9.attention.wk.weight": "model-00007-of-00007.safetensors",
571
+ "model.vision_backbone.image_vit.blocks.9.attention.wo.bias": "model-00007-of-00007.safetensors",
572
+ "model.vision_backbone.image_vit.blocks.9.attention.wo.weight": "model-00007-of-00007.safetensors",
573
+ "model.vision_backbone.image_vit.blocks.9.attention.wq.bias": "model-00007-of-00007.safetensors",
574
+ "model.vision_backbone.image_vit.blocks.9.attention.wq.weight": "model-00007-of-00007.safetensors",
575
+ "model.vision_backbone.image_vit.blocks.9.attention.wv.bias": "model-00007-of-00007.safetensors",
576
+ "model.vision_backbone.image_vit.blocks.9.attention.wv.weight": "model-00007-of-00007.safetensors",
577
+ "model.vision_backbone.image_vit.blocks.9.attention_norm.bias": "model-00007-of-00007.safetensors",
578
+ "model.vision_backbone.image_vit.blocks.9.attention_norm.weight": "model-00007-of-00007.safetensors",
579
+ "model.vision_backbone.image_vit.blocks.9.feed_forward.w1.bias": "model-00007-of-00007.safetensors",
580
+ "model.vision_backbone.image_vit.blocks.9.feed_forward.w1.weight": "model-00007-of-00007.safetensors",
581
+ "model.vision_backbone.image_vit.blocks.9.feed_forward.w2.bias": "model-00007-of-00007.safetensors",
582
+ "model.vision_backbone.image_vit.blocks.9.feed_forward.w2.weight": "model-00007-of-00007.safetensors",
583
+ "model.vision_backbone.image_vit.blocks.9.ffn_norm.bias": "model-00007-of-00007.safetensors",
584
+ "model.vision_backbone.image_vit.blocks.9.ffn_norm.weight": "model-00007-of-00007.safetensors",
585
+ "model.vision_backbone.image_vit.class_embedding": "model-00007-of-00007.safetensors",
586
+ "model.vision_backbone.image_vit.patch_embedding.weight": "model-00007-of-00007.safetensors",
587
+ "model.vision_backbone.image_vit.positional_embedding": "model-00007-of-00007.safetensors",
588
+ "model.vision_backbone.image_vit.pre_ln.bias": "model-00007-of-00007.safetensors",
589
+ "model.vision_backbone.image_vit.pre_ln.weight": "model-00007-of-00007.safetensors",
590
+ "model.vision_backbone.pad_embed": "model-00007-of-00007.safetensors"
591
+ }
592
+ }
modeling_molmo.py ADDED
@@ -0,0 +1,1409 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from copy import deepcopy
3
+ from dataclasses import fields, dataclass, replace
4
+ from enum import Enum
5
+ from typing import List, Optional, Tuple, Union, Dict, Any, Sequence, Callable, cast, MutableMapping
6
+
7
+ import torch
8
+ from transformers import PreTrainedModel, GenerationConfig, add_start_docstrings
9
+ from transformers.activations import ACT2FN
10
+ from transformers.cache_utils import Cache
11
+ from transformers.modeling_flash_attention_utils import _flash_attention_forward
12
+ from transformers.modeling_outputs import CausalLMOutputWithPast, ModelOutput
13
+ from transformers.models.auto import AutoModelForCausalLM
14
+ from torch import nn
15
+ from transformers.utils import logging
16
+
17
+ from .config_molmo import MolmoConfig, MolmoVisionConfig
18
+ from torch.nn import functional as F
19
+
20
+
21
+ logger = logging.get_logger(__name__)
22
+
23
+
24
+ MOLMO_START_DOCSTRING = r"""
25
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
26
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
27
+ etc.)
28
+
29
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
30
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
31
+ and behavior.
32
+
33
+ Parameters:
34
+ config ([`MolmoConfig`]):
35
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
36
+ load the weights associated with the model, only the configuration. Check out the
37
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
38
+ """
39
+
40
+
41
+ @add_start_docstrings(
42
+ "The bare Molmo Model outputting raw hidden-states without any specific head on top.",
43
+ MOLMO_START_DOCSTRING,
44
+ )
45
+ class MolmoPreTrainedModel(PreTrainedModel):
46
+ config_class = MolmoConfig
47
+ base_model_prefix = "model"
48
+ _no_split_modules = ["MolmoBlock", "MolmoeBlock", "MolmoVisionBlock"]
49
+ _skip_keys_device_placement = "past_key_values"
50
+ _supports_flash_attn_2 = True
51
+ _supports_sdpa = True
52
+ # supports_gradient_checkpointing = True
53
+ # _supports_cache_class = True
54
+ # _supports_static_cache = False
55
+
56
+ def _init_weights(self, module):
57
+ std = self.config.initializer_range
58
+ if isinstance(module, (nn.Linear,)):
59
+ module.weight.data.normal_(mean=0.0, std=std)
60
+ if module.bias is not None:
61
+ module.bias.data.zero_()
62
+ elif isinstance(module, nn.Embedding):
63
+ module.weight.data.normal_(mean=0.0, std=std)
64
+
65
+
66
+ class MolmoRotaryEmbedding(nn.Module):
67
+ """
68
+ [Rotary positional embeddings (RoPE)](https://arxiv.org/abs/2104.09864).
69
+ """
70
+
71
+ def __init__(self, dim, max_position_embeddings=2048, rope_theta=10000, full_precision=True, device=None):
72
+ super().__init__()
73
+ self.dim = dim
74
+ self.rope_theta = rope_theta
75
+ self.full_precision = full_precision
76
+ self.max_position_embeddings = max_position_embeddings
77
+
78
+ # Cache sin/cos embeddings
79
+ dim = self.dim
80
+ inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float) / dim))
81
+ seq = torch.arange(self.max_position_embeddings, device=device, dtype=torch.float)
82
+ freqs = torch.einsum("i , j -> i j", seq, inv_freq)
83
+ positions = torch.cat((freqs, freqs), dim=-1)
84
+ pos_sin, pos_cos = positions.sin()[None, None, :, :], positions.cos()[None, None, :, :]
85
+ self.register_buffer("rope_pos_sin", pos_sin, persistent=False)
86
+ self.register_buffer("rope_pos_cos", pos_cos, persistent=False)
87
+
88
+ def rotate_half(self, x: torch.Tensor) -> torch.Tensor:
89
+ B, nh, T, hs = x.size()
90
+ x = x.view(B, nh, T, 2, hs // 2)
91
+ x1, x2 = x.unbind(dim=-2)
92
+ return torch.cat((-x2, x1), dim=-1)
93
+
94
+ def apply_rotary_pos_emb(self, pos_sin: torch.Tensor, pos_cos: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
95
+ return (t * pos_cos) + (self.rotate_half(t) * pos_sin)
96
+
97
+ def forward(
98
+ self,
99
+ q: torch.Tensor,
100
+ k: torch.Tensor,
101
+ position_ids: Optional[torch.Tensor] = None
102
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
103
+ if self.full_precision:
104
+ q_, k_ = q.float(), k.float()
105
+ else:
106
+ q_, k_ = q, k
107
+
108
+ with torch.autocast(q.device.type, enabled=False):
109
+ batch_size = q_.shape[0]
110
+ query_len, key_len = q_.shape[-2], k_.shape[-2] # could be different if layer_past not None
111
+ if position_ids is not None:
112
+ freqs_cis_len = self.max_position_embeddings
113
+ else:
114
+ freqs_cis_len = key_len
115
+ # self.get_rotary_embedding(freqs_cis_len, q_.device)
116
+ pos_sin = self.rope_pos_sin[:, :, :freqs_cis_len, :].type_as(q_)
117
+ pos_cos = self.rope_pos_cos[:, :, :freqs_cis_len, :].type_as(q_)
118
+ if position_ids is not None:
119
+ assert query_len == key_len, "Query and key lengths must be equal when using position IDs."
120
+ pos_sin = pos_sin[0, 0][position_ids].view(
121
+ (batch_size, 1, key_len, pos_sin.shape[-1])
122
+ )
123
+ pos_cos = pos_cos[0, 0][position_ids].view(
124
+ (batch_size, 1, key_len, pos_cos.shape[-1])
125
+ )
126
+ q_ = self.apply_rotary_pos_emb(
127
+ pos_sin[:, :, key_len - query_len : key_len, :],
128
+ pos_cos[:, :, key_len - query_len : key_len, :],
129
+ q_,
130
+ )
131
+ k_ = self.apply_rotary_pos_emb(pos_sin, pos_cos, k_)
132
+ return q_.type_as(q), k_.type_as(k)
133
+
134
+
135
+ class MolmoAttention(nn.Module):
136
+ def __init__(
137
+ self,
138
+ config: MolmoConfig,
139
+ device=None
140
+ ):
141
+ super().__init__()
142
+ self.config = config
143
+ self.rotary_emb = MolmoRotaryEmbedding(
144
+ config.hidden_size // config.num_attention_heads,
145
+ config.max_position_embeddings,
146
+ config.rope_theta, device=device)
147
+
148
+ self.k_norm: Optional[nn.Module] = None
149
+ self.q_norm: Optional[nn.Module] = None
150
+ self.hidden_size = config.intermediate_size
151
+ if config.qk_layer_norm:
152
+ if config.num_key_value_heads is None:
153
+ config.num_key_value_heads = config.num_attention_heads
154
+ self.q_norm = MolmoRmsLayerNorm(
155
+ config,
156
+ size=config.hidden_size,
157
+ eps=config.layer_norm_eps
158
+ )
159
+ self.k_norm = MolmoRmsLayerNorm(
160
+ config,
161
+ size=config.hidden_size,
162
+ eps=config.layer_norm_eps
163
+ )
164
+
165
+ # Attention output projection.
166
+ input_dim = config.hidden_size
167
+ head_dim = config.hidden_size // config.num_attention_heads
168
+ self.fused_dims = (
169
+ config.hidden_size,
170
+ config.num_key_value_heads * head_dim,
171
+ config.num_key_value_heads * head_dim,
172
+ )
173
+ self.att_proj = nn.Linear(
174
+ config.hidden_size, sum(self.fused_dims),
175
+ bias=config.qkv_bias,
176
+ )
177
+ self.attn_out = nn.Linear(
178
+ input_dim, config.hidden_size,
179
+ bias=False,
180
+ )
181
+
182
+ def attention(self,
183
+ q: torch.Tensor,
184
+ k: torch.Tensor,
185
+ v: torch.Tensor,
186
+ attention_mask: Optional[torch.Tensor] = None,
187
+ position_ids: Optional[torch.Tensor] = None,
188
+ drop_mask: Optional[torch.Tensor] = None,
189
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
190
+ use_cache: bool = False,
191
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
192
+ B, T, C = q.size() # batch size, sequence length, hidden_size
193
+ dtype = k.dtype
194
+
195
+ # Optionally apply layer norm to keys and queries.
196
+ if self.q_norm is not None and self.k_norm is not None:
197
+ q = self.q_norm(q).to(dtype=dtype)
198
+ k = self.k_norm(k).to(dtype=dtype)
199
+
200
+ # Move head forward to be next to the batch dim.
201
+ # shape: (B, nh, T, hs)
202
+ q = q.view(B, T, self.config.num_attention_heads, C // self.config.num_attention_heads).transpose(1, 2)
203
+ # shape: (B, n_kv_h, T, hs)
204
+ k = k.view(B, T, self.config.num_key_value_heads, C // self.config.num_attention_heads).transpose(1, 2)
205
+ # shape: (B, n_kv_h, T, hs)
206
+ v = v.view(B, T, self.config.num_key_value_heads, C // self.config.num_attention_heads).transpose(1, 2)
207
+
208
+ # Apply rotary embeddings
209
+ q, k = self.rotary_emb(q, k, position_ids=position_ids)
210
+
211
+ if layer_past is not None:
212
+ past_key, past_value = layer_past
213
+ k = torch.cat((past_key.to(k.device), k), dim=-2)
214
+ v = torch.cat((past_value.to(v.device), v), dim=-2)
215
+
216
+ present = (k, v) if use_cache else None
217
+ query_len, key_len = q.shape[-2], k.shape[-2] # could be different if layer_past not None
218
+
219
+ if attention_mask is not None:
220
+ attention_mask = attention_mask[:, :, key_len - query_len: key_len, :key_len]
221
+
222
+ # if attention_bias is not None:
223
+ # attention_bias = self._cast_attn_bias(
224
+ # attention_bias[:, :, key_len - query_len : key_len, :key_len], dtype)
225
+
226
+ # Get the attention scores.
227
+ # shape: (B, nh, T, hs)
228
+ att = self._scaled_dot_product_attention(
229
+ q,
230
+ k,
231
+ v,
232
+ attention_mask=attention_mask,
233
+ dropout_p=0.0 if not self.training else self.config.attention_dropout,
234
+ is_causal=attention_mask is None,
235
+ )
236
+
237
+ # Re-assemble all head outputs side-by-side.
238
+ att = att.transpose(1, 2).contiguous().view(B, T, C)
239
+
240
+ # Apply output projection.
241
+ return self.attn_out(att), present
242
+
243
+ def _scaled_dot_product_attention(
244
+ self,
245
+ q: torch.Tensor,
246
+ k: torch.Tensor,
247
+ v: torch.Tensor,
248
+ attention_mask: Optional[torch.Tensor] = None,
249
+ dropout_p: float = 0.0,
250
+ is_causal: bool = False,
251
+ ) -> torch.Tensor:
252
+ if attention_mask is not None:
253
+ attention_mask = attention_mask.to(q.device)
254
+
255
+ if self.config.attention_type == "sdpa":
256
+ assert k.size(1) == v.size(1)
257
+ num_kv_heads = k.size(1)
258
+ num_q_heads = q.size(1)
259
+ if num_q_heads != num_kv_heads:
260
+ assert num_q_heads % num_kv_heads == 0
261
+ k = k.repeat_interleave(num_q_heads // num_kv_heads, dim=1, output_size=num_q_heads)
262
+ v = v.repeat_interleave(num_q_heads // num_kv_heads, dim=1, output_size=num_q_heads)
263
+
264
+ return F.scaled_dot_product_attention(
265
+ q,
266
+ k,
267
+ v,
268
+ attn_mask=attention_mask,
269
+ dropout_p=dropout_p,
270
+ is_causal=is_causal,
271
+ )
272
+ elif self.config.attention_type == "flash":
273
+ # Downcast in case we are running with fp32 hidden states
274
+ # Our attention mask is [1, 1, N, N]
275
+ valid_mask = torch.reduce_any(attention_mask, -1)[0]
276
+ attn_output = _flash_attention_forward(
277
+ q.transpose(1, 2).to(torch.bfloat16),
278
+ k.transpose(1, 2).to(torch.bfloat16),
279
+ v.transpose(1, 2).to(torch.bfloat16),
280
+ attention_mask=valid_mask,
281
+ query_length=q.shape[2],
282
+ is_causal=True,
283
+ )
284
+ else:
285
+ raise NotImplementedError(self.config.attention_type)
286
+
287
+ def forward(
288
+ self,
289
+ x,
290
+ attention_mask,
291
+ position_ids,
292
+ layer_past,
293
+ use_cache
294
+ ):
295
+ qkv = self.att_proj(x)
296
+
297
+ q, k, v = qkv.split(self.fused_dims, dim=-1)
298
+
299
+ # Get attention scores.
300
+ att, cache = self.attention(
301
+ q, k, v,
302
+ attention_mask,
303
+ position_ids=position_ids,
304
+ layer_past=layer_past,
305
+ use_cache=use_cache
306
+ )
307
+ return att, cache
308
+
309
+
310
+ class MolmoMlp(nn.Module):
311
+ def __init__(self, input_dim, hidden_size, activation_fn, include_bias=False):
312
+ super().__init__()
313
+ self.ff_proj = nn.Linear(input_dim, hidden_size, bias=include_bias)
314
+ self.ff_out = nn.Linear(hidden_size//2, input_dim, bias=include_bias)
315
+ self.act = ACT2FN[activation_fn]
316
+
317
+ def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
318
+ x = self.ff_proj(x)
319
+ x, gate = x.chunk(2, dim=-1)
320
+ x = self.act(gate) * x
321
+ x = self.ff_out(x)
322
+ return x
323
+
324
+
325
+ class MolmoBlock(nn.Module):
326
+ def __init__(self, config: MolmoConfig, device=None):
327
+ super().__init__()
328
+ self.config = config
329
+ self.hidden_size = config.intermediate_size
330
+ self.dropout = nn.Dropout(config.residual_dropout)
331
+ self.attn = MolmoAttention(config)
332
+ self.attn_norm = MolmoRmsLayerNorm(config, size=config.hidden_size, eps=config.layer_norm_eps)
333
+ self.mlp = MolmoMlp(config.hidden_size, config.intermediate_size, config.activation_type)
334
+ self.ff_norm = MolmoRmsLayerNorm(config)
335
+
336
+ def forward(
337
+ self,
338
+ x: torch.Tensor,
339
+ attention_mask: Optional[torch.Tensor] = None,
340
+ position_ids: Optional[torch.Tensor] = None,
341
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
342
+ use_cache: bool = False,
343
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
344
+ if not self.config.norm_after:
345
+ atten_in = self.attn_norm(x)
346
+ else:
347
+ atten_in = x
348
+
349
+ att, cache = self.attn(
350
+ atten_in,
351
+ attention_mask=attention_mask,
352
+ position_ids=position_ids,
353
+ layer_past=layer_past,
354
+ use_cache=use_cache
355
+ )
356
+
357
+ if self.config.norm_after:
358
+ att = self.attn_norm(att)
359
+
360
+ x = x + self.dropout(att)
361
+
362
+ og_x = x
363
+
364
+ if not self.config.norm_after:
365
+ x = self.ff_norm(x)
366
+
367
+ x = self.mlp(x)
368
+
369
+ if self.config.norm_after:
370
+ x = self.ff_norm(x)
371
+
372
+ x = self.dropout(x)
373
+ x = og_x + x
374
+
375
+ return x, cache
376
+
377
+
378
+ class MolmoeMLP(nn.Module):
379
+ def __init__(self, input_dim, hidden_size, activation):
380
+ super().__init__()
381
+ self.gate_proj = nn.Linear(input_dim, hidden_size, bias=False)
382
+ self.up_proj = nn.Linear(input_dim, hidden_size, bias=False)
383
+ self.down_proj = nn.Linear(hidden_size, input_dim, bias=False)
384
+ self.act_fn = ACT2FN[activation]
385
+
386
+ def forward(self, x):
387
+ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
388
+
389
+
390
+ class MolmoeMlpExpert(nn.Module):
391
+ def __init__(self, config):
392
+ super().__init__()
393
+ self.num_experts = config.moe_num_experts
394
+ self.top_k = config.moe_top_k
395
+ self.gate = nn.Linear(config.hidden_size, self.num_experts, bias=False)
396
+ self.experts = nn.ModuleList([MolmoeMLP(config.hidden_size, config.intermediate_size // 2, config.activation_type)
397
+ for _ in range(self.num_experts)])
398
+
399
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
400
+ # hidden_states = self.ff_norm(hidden_states)
401
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
402
+ hidden_states = hidden_states.view(-1, hidden_dim)
403
+ # router_logits: (batch * sequence_length, n_experts)
404
+ router_logits = self.gate(hidden_states)
405
+
406
+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
407
+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
408
+
409
+ # we cast back to the input dtype
410
+ routing_weights = routing_weights.to(hidden_states.dtype)
411
+
412
+ final_hidden_states = torch.zeros(
413
+ (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device
414
+ )
415
+
416
+ # One hot encode the selected experts to create an expert mask
417
+ # this will be used to easily index which expert is going to be selected
418
+ expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
419
+
420
+ # Loop over all available experts in the model and perform the computation on each expert
421
+ for expert_idx in range(self.num_experts):
422
+ expert_layer = self.experts[expert_idx]
423
+ idx, top_x = torch.where(expert_mask[expert_idx])
424
+
425
+ # Index the correct hidden states and compute the expert hidden state for
426
+ # the current expert. We need to make sure to multiply the output hidden
427
+ # states by `routing_weights` on the corresponding tokens (top-1 and top-2)
428
+ current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
429
+ current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None]
430
+
431
+ # However `index_add_` only support torch tensors for indexing so we'll use
432
+ # the `top_x` tensor here.
433
+ final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))
434
+ final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
435
+ return final_hidden_states, router_logits
436
+
437
+
438
+ class MolmoeBlock(nn.Module):
439
+ def __init__(self, config: MolmoConfig):
440
+ super().__init__()
441
+ self.attn = MolmoAttention(config)
442
+ self.attn_norm = MolmoRmsLayerNorm(config, size=config.hidden_size, eps=config.layer_norm_eps)
443
+ assert config.moe_num_experts > 0
444
+ self.ff_norm = MolmoRmsLayerNorm(config, size=config.hidden_size, eps=config.layer_norm_eps)
445
+ self.mlp = MolmoeMlpExpert(config)
446
+ self.config = config
447
+ self.hidden_size = config.intermediate_size
448
+ self.dropout = nn.Dropout(config.residual_dropout)
449
+
450
+ def forward(
451
+ self,
452
+ x: torch.Tensor,
453
+ attention_mask: Optional[torch.FloatTensor] = None,
454
+ position_ids: Optional[torch.Tensor] = None,
455
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
456
+ use_cache: bool = False,
457
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
458
+ if not self.config.norm_after:
459
+ atten_in = self.attn_norm(x)
460
+ else:
461
+ atten_in = x
462
+
463
+ att, cache = self.attn(
464
+ atten_in,
465
+ attention_mask=attention_mask,
466
+ position_ids=position_ids,
467
+ layer_past=layer_past,
468
+ use_cache=use_cache
469
+ )
470
+
471
+ if self.config.norm_after:
472
+ att = self.attn_norm(att)
473
+
474
+ x = x + self.dropout(att)
475
+ og_x = x
476
+
477
+ if not self.config.norm_after:
478
+ x = self.ff_norm(x)
479
+
480
+ x, _ = self.mlp(x)
481
+
482
+ if self.config.norm_after:
483
+ x = self.ff_norm(x)
484
+
485
+ x = self.dropout(x)
486
+ x = og_x + x
487
+ return x, cache
488
+
489
+
490
+ class Embedding(nn.Module):
491
+ def __init__(
492
+ self,
493
+ num_embeddings: int,
494
+ num_new_embeddings: int,
495
+ features: int,
496
+ device: Union[str, torch.device] = None,
497
+ initializer_range: float = 0.02,
498
+ new_embed_initializer_range: float = 0.02,
499
+ ):
500
+ super().__init__()
501
+ self.initializer_range = initializer_range
502
+ self.new_embed_initializer_range = new_embed_initializer_range
503
+ self.embedding = nn.Parameter(
504
+ torch.zeros(num_embeddings, features, device=device),
505
+ )
506
+ # We keep the special token embedding separate from the embedding from the LM so we can
507
+ # put a separate learning rate of them during training
508
+ self.new_embedding = nn.Parameter(torch.zeros(num_new_embeddings, features, device=device))
509
+
510
+ def reset_parameters(self):
511
+ nn.init.normal_(self.embedding, std=self.initializer_range)
512
+ nn.init.normal_(self.new_embedding, std=self.new_embed_initializer_range)
513
+
514
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
515
+ return F.embedding(x, torch.cat([self.embedding, self.new_embedding], dim=0))
516
+
517
+
518
+ def _expand_token(token, batch_size: int):
519
+ return token.view(1, 1, -1).expand(batch_size, -1, -1)
520
+
521
+
522
+ class VisionMlp(nn.Module):
523
+ def __init__(self, dim: int, hidden_dim: int, hidden_act: str, device=None):
524
+ super().__init__()
525
+ self.w1 = nn.Linear(dim, hidden_dim, bias=True, device=device)
526
+ self.act = ACT2FN[hidden_act]
527
+ self.w2 = nn.Linear(hidden_dim, dim, bias=True, device=device)
528
+
529
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
530
+ return self.w2(self.act(self.w1(x)))
531
+
532
+
533
+ class MolmoVisionBlock(nn.Module):
534
+
535
+ def __init__(self, config: MolmoVisionConfig, attention_type, device=None):
536
+ super().__init__()
537
+ self.attention = VisionAttention(config, device=device, attention_type=attention_type)
538
+ self.feed_forward = VisionMlp(
539
+ config.image_emb_dim, config.image_mlp_dim, config.image_mlp_activations, device)
540
+ self.attention_norm = nn.LayerNorm(
541
+ config.image_emb_dim,
542
+ eps=config.image_norm_eps,
543
+ device=device,
544
+ )
545
+ self.ffn_norm = nn.LayerNorm(
546
+ config.image_emb_dim,
547
+ eps=config.image_norm_eps,
548
+ device=device,
549
+ )
550
+
551
+ def reset_parameters(self):
552
+ self.attention.reset_parameters()
553
+ self.feed_forward.reset_parameters()
554
+ self.attention_norm.reset_parameters()
555
+ self.ffn_norm.reset_parameters()
556
+
557
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
558
+ x = x + self.attention(self.attention_norm(x))
559
+ x = x + self.feed_forward(self.ffn_norm(x))
560
+ return x
561
+
562
+
563
+ class VisionPreLayerNorm(nn.LayerNorm):
564
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
565
+ orig_type = x.dtype
566
+ x = F.layer_norm(x.to(torch.float32), self.normalized_shape, self.weight.to(torch.float32),
567
+ self.bias.to(torch.float32), self.eps)
568
+ return x.to(orig_type)
569
+
570
+
571
+ class VisionTransformer(nn.Module):
572
+
573
+ def __init__(self, config: MolmoVisionConfig, attention_type, device=None):
574
+ super().__init__()
575
+ self.config = config
576
+
577
+ # class embeddings and positional embeddings
578
+ self.scale = config.image_emb_dim ** -0.5
579
+ self.class_embedding = nn.Parameter(
580
+ torch.zeros(config.image_emb_dim, device=device))
581
+ self.positional_embedding = nn.Parameter(
582
+ torch.zeros(config.image_num_pos, config.image_emb_dim, device=device))
583
+
584
+ image_patch_size = config.image_patch_size
585
+ self.patch_embedding = nn.Linear(
586
+ image_patch_size * image_patch_size * 3,
587
+ config.image_emb_dim,
588
+ bias=False,
589
+ device=device
590
+ )
591
+
592
+ self.pre_ln = VisionPreLayerNorm(
593
+ config.image_emb_dim,
594
+ eps=config.image_norm_eps,
595
+ )
596
+ self.blocks = nn.ModuleList([
597
+ MolmoVisionBlock(config, attention_type=attention_type, device=device)
598
+ for _ in range(config.image_num_layers)
599
+ ])
600
+
601
+ def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor:
602
+ cls_emb = self.positional_embedding[0:1]
603
+ pos_emb = self.positional_embedding[1:]
604
+
605
+ pos_emb = pos_emb.reshape(
606
+ (int(math.sqrt(pos_emb.shape[0])), int(math.sqrt(pos_emb.shape[0])), pos_emb.shape[1])
607
+ )
608
+
609
+ (patch_num_0, patch_num_1) = patch_num
610
+
611
+ if pos_emb.shape[0] != patch_num_0 or pos_emb.shape[1] != patch_num_1:
612
+ # Dervied from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
613
+ # antialias: default True in jax.image.resize
614
+ pos_emb = pos_emb.unsqueeze(0).permute(0, 3, 1, 2)
615
+ pos_emb = F.interpolate(
616
+ pos_emb, size=(patch_num_0, patch_num_1), mode="bicubic", align_corners=False, antialias=True,
617
+ )
618
+ pos_emb = pos_emb.permute(0, 2, 3, 1).squeeze(0)
619
+
620
+ pos_emb = pos_emb.reshape(-1, pos_emb.shape[-1])
621
+ x = x + torch.cat([cls_emb[None, :, :], pos_emb[None, :, :]], dim=1).to(x.dtype)
622
+ return x
623
+
624
+ def forward(self, x: torch.Tensor, patch_num: int = None) -> List[torch.Tensor]:
625
+ if patch_num is None:
626
+ patch_num = self.config.image_num_patch
627
+ B, N, D = x.shape
628
+
629
+ x = self.patch_embedding(x)
630
+
631
+ # class embeddings and positional embeddings
632
+ x = torch.cat([_expand_token(self.class_embedding, x.shape[0]).to(x.dtype), x], dim=1)
633
+ x = self.add_pos_emb(x, patch_num)
634
+
635
+ x = self.pre_ln(x)
636
+
637
+ hidden_states = []
638
+ for r in self.blocks:
639
+ x = r(x)
640
+ hidden_states.append(x)
641
+ return hidden_states
642
+
643
+
644
+ class VisionAttention(nn.Module):
645
+ def __init__(self, config: MolmoVisionConfig, use_bias: bool =True,
646
+ embed_dim: int=None, device=None, attention_type: str="sdpa"):
647
+ super().__init__()
648
+ self.config = config
649
+ self.embed_dim = config.image_emb_dim
650
+ self.num_heads = config.image_num_heads
651
+ self.head_dim = config.image_head_dim
652
+ self.num_key_value_heads = config.image_num_key_value_heads
653
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
654
+ self.initializer_range = config.initializer_range
655
+ self.attention_type = attention_type
656
+
657
+ embed_dim = embed_dim if embed_dim else config.image_emb_dim
658
+
659
+ self.wq = nn.Linear(
660
+ embed_dim,
661
+ self.num_heads * self.head_dim,
662
+ bias=use_bias,
663
+ device=device,
664
+ )
665
+ self.wk = nn.Linear(
666
+ embed_dim,
667
+ self.num_key_value_heads * self.head_dim,
668
+ bias=use_bias,
669
+ device=device,
670
+ )
671
+ self.wv = nn.Linear(
672
+ embed_dim,
673
+ self.num_key_value_heads * self.head_dim,
674
+ bias=use_bias,
675
+ device=device,
676
+ )
677
+ self.wo = nn.Linear(
678
+ self.num_heads * self.head_dim,
679
+ self.embed_dim,
680
+ bias=use_bias,
681
+ device=device,
682
+ )
683
+ self.residual_dropout = nn.Dropout(config.residual_dropout)
684
+
685
+ def _split_heads(self, hidden_states, num_heads) -> torch.Tensor:
686
+ return hidden_states.reshape(hidden_states.shape[:2] + (num_heads, self.head_dim))
687
+
688
+ def _merge_heads(self, hidden_states) -> torch.Tensor:
689
+ return hidden_states.reshape(hidden_states.shape[:2] + (self.embed_dim,))
690
+
691
+ def forward(self, inputs_q: torch.Tensor, inputs_kv: Optional[torch.Tensor] = None) -> torch.Tensor:
692
+ if inputs_kv is not None:
693
+ inputs_k = inputs_kv
694
+ inputs_v = inputs_kv
695
+ else:
696
+ inputs_k = inputs_q
697
+ inputs_v = inputs_q
698
+
699
+ xq, xk, xv = self.wq(inputs_q), self.wk(inputs_k), self.wv(inputs_v)
700
+
701
+ xq = self._split_heads(xq, self.num_heads)
702
+ xk = self._split_heads(xk, self.num_key_value_heads)
703
+ xv = self._split_heads(xv, self.num_key_value_heads)
704
+
705
+ if self.num_heads != self.num_key_value_heads:
706
+ xk = xk.repeat_interleave(self.num_key_value_groups, dim=2, output_size=self.num_heads)
707
+ xv = xv.repeat_interleave(self.num_key_value_groups, dim=2, output_size=self.num_heads)
708
+
709
+ og_dtype = xq.dtype
710
+
711
+ if self.config.float32_attention:
712
+ xq = xq.to(torch.float)
713
+ xk = xk.to(torch.float)
714
+
715
+ if self.attention_type == "direct":
716
+ attn_weights = torch.einsum("...qhd,...khd->...hqk", xq / math.sqrt(xq.size(-1)), xk)
717
+ attn_weights = F.softmax(attn_weights, dim=-1)
718
+ attn_output = torch.einsum("...hqk,...khd->...qhd", attn_weights.to(xv.dtype), xv)
719
+
720
+ elif self.attention_type == "sdpa":
721
+ if self.config.float32_attention and not torch.is_autocast_enabled():
722
+ xv = xv.to(torch.float32)
723
+ attn_output = F.scaled_dot_product_attention(
724
+ xq.transpose(1, 2).contiguous(),
725
+ xk.transpose(1, 2).contiguous(),
726
+ xv.transpose(1, 2).contiguous(),
727
+ is_causal=False,
728
+ ).transpose(1, 2)
729
+
730
+ elif self.attention_type == "flash":
731
+ assert not self.config.float32_attention
732
+ # Downcast in case we are running with fp32 hidden states
733
+ attn_output = _flash_attention_forward(
734
+ xq.transpose(1, 2).to(torch.bfloat16),
735
+ xk.transpose(1, 2).to(torch.bfloat16),
736
+ xv.transpose(1, 2).to(torch.bfloat16),
737
+ attention_mask=None,
738
+ query_length=inputs_q.shape[1],
739
+ is_causal=False,
740
+ )
741
+ else:
742
+ raise NotImplementedError(self.attention_type)
743
+ attn_output = attn_output.to(og_dtype)
744
+ attn_output = self._merge_heads(attn_output)
745
+ attn_output = self.wo(attn_output)
746
+ attn_output = self.residual_dropout(attn_output)
747
+ return attn_output
748
+
749
+
750
+ class MolmoImageProjector(nn.Module):
751
+ def __init__(self, input_dim: int, hidden_dim, output_dim, act_fn="silu", device=None):
752
+ super().__init__()
753
+ self.w1 = nn.Linear(input_dim, hidden_dim, bias=False, device=device)
754
+ self.w2 = nn.Linear(hidden_dim, output_dim, bias=False, device=device)
755
+ self.w3 = nn.Linear(input_dim, hidden_dim, bias=False, device=device)
756
+ self.act_fn = ACT2FN[act_fn]
757
+
758
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
759
+ return self.w2(self.act_fn(self.w1(x))*self.w3(x))
760
+
761
+
762
+ class OLMoVisionBackbone(nn.Module):
763
+ def __init__(self, config: MolmoConfig):
764
+ super().__init__()
765
+ self.config = config
766
+ self.image_vit = VisionTransformer(config.vision_config, config.attention_type)
767
+
768
+ self.image_pooling_2d = VisionAttention(
769
+ config.vision_config,
770
+ embed_dim=len(config.vit_layers)*config.vision_config.image_emb_dim,
771
+ attention_type=config.attention_type
772
+ )
773
+
774
+ # `MLP` assume the activation takes two inputs, so it must be a 'llama' version
775
+ if config.activation_type == "swiglu":
776
+ mlp_config = replace(config, activation_type="llama_swiglu")
777
+ elif config.activation_type == "gelu":
778
+ raise NotImplementedError()
779
+ else:
780
+ mlp_config = config
781
+
782
+ self.image_projector = MolmoImageProjector(
783
+ config.vision_config.image_emb_dim,
784
+ config.intermediate_size//2, # //2 since `mlp_hidden_size` includes the gate and parts
785
+ config.hidden_size,
786
+ act_fn=config.activation_type
787
+ )
788
+ self.image_feature_dropout = nn.Dropout(config.image_feature_dropout)
789
+ self.num_prefix_tokens = 1
790
+
791
+ self.pad_embed = None
792
+ if config.image_padding_embed:
793
+ image_dim = config.vision_config.image_emb_dim*len(self.config.vit_layers)
794
+ if config.image_padding_embed == "pad_and_partial_pad":
795
+ self.pad_embed = nn.Parameter(torch.zeros((2, image_dim)))
796
+ else:
797
+ raise ValueError(config.image_padding_embed)
798
+
799
+ def encode_image(self, images: torch.Tensor) -> torch.Tensor:
800
+ cfg = self.config
801
+ v_cfg = self.config.vision_config
802
+ B, T, N, D = images.shape
803
+
804
+ mask = ~torch.all(images.view(B * T, N, D) == -1, dim=(1, 2), keepdim=True)
805
+
806
+ # Output all hidden states
807
+ # n_layers x (batch_num_crops, (1+)n_tokens, image_emb_dim)
808
+ images = images.view(B * T, N, D)
809
+ image_features = self.image_vit(images)
810
+
811
+ if cfg.vit_layers is not None:
812
+ features = []
813
+ for layer in cfg.vit_layers:
814
+ features.append(image_features[layer])
815
+ image_features = torch.cat(features, dim=-1)
816
+ else:
817
+ image_features = image_features[-1]
818
+
819
+ cls_embed: torch.Tensor = None
820
+ if self.num_prefix_tokens > 0:
821
+ cls_embed = image_features[:, 0]
822
+ image_features = image_features[:, 1:]
823
+
824
+ image_features = image_features * mask
825
+ image_features = image_features.view(B, T, N, -1)
826
+
827
+ cls_embed = cls_embed.view(B, T, -1) if cls_embed is not None else None
828
+
829
+ return image_features, cls_embed
830
+
831
+ def forward(self, images: torch.Tensor, image_masks: torch.Tensor) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
832
+ cfg = self.config
833
+
834
+ # image_features: (batch_size, num_crops(=num_image), num_patch, nximage_emb_dim)
835
+ batch_size, num_image = images.shape[:2]
836
+ image_features, cls_embed = self.encode_image(images)
837
+
838
+ if cfg.image_padding_embed:
839
+ assert image_masks is not None
840
+ if cfg.image_padding_embed == "pad_embed":
841
+ all_pad = (image_masks == 0).to(dtype=torch.float32)
842
+ pad_embed = self.pad_embed[None, None, None, :]
843
+ image_features = image_features + pad_embed * torch.unsqueeze(all_pad, -1)
844
+ elif cfg.image_padding_embed == "regress":
845
+ pad_embed = self.pad_embed[None, None, None, :]
846
+ image_features = image_features + pad_embed * torch.unsqueeze(torch.maximum(image_masks, torch.zeros_like(image_masks)), -1)
847
+ elif cfg.image_padding_embed == "pad_and_partial_pad":
848
+ pad_embed = self.pad_embed[:, None, None, None, :]
849
+ all_pad = image_masks == 0
850
+ partial_pad = torch.logical_and(image_masks < 1, torch.logical_not(all_pad)).to(dtype=image_features.dtype)
851
+ all_pad = all_pad.to(dtype=image_features.dtype)
852
+ image_features = image_features + pad_embed[0] * torch.unsqueeze(all_pad, -1)
853
+ image_features = image_features + pad_embed[1] * torch.unsqueeze(partial_pad, -1)
854
+ else:
855
+ raise ValueError(cfg.image_padding_embed)
856
+
857
+ image_features = self.image_feature_dropout(image_features)
858
+ if cls_embed is not None:
859
+ cls_embed = self.image_feature_dropout(cls_embed)
860
+
861
+ image_features = image_features.reshape(
862
+ (batch_size, num_image) + cfg.image_num_patch + (-1,))
863
+
864
+ # transpose to get 2x2 feature squares [n_patches, 4, n_features]
865
+ batch, n_crops, h, w, c = image_features.shape
866
+ image_features = torch.reshape(image_features, [batch*n_crops, h//2, 2, w//2, 2, c])
867
+ image_features = torch.permute(image_features, [0, 1, 3, 2, 4, 5])
868
+ image_features = torch.reshape(image_features, [batch*n_crops*h//2*w//2, 2*2, c])
869
+
870
+ query = image_features.mean(-2, keepdim=True)
871
+ image_features = self.image_pooling_2d(query, image_features)
872
+
873
+ h = self.config.vision_config.image_num_patch[0]//2
874
+ w = self.config.vision_config.image_num_patch[1]//2
875
+ image_features = image_features.reshape(batch_size, num_image, h * w, -1)
876
+
877
+ # MLP layer to map the feature.
878
+ image_features = self.image_projector(image_features)
879
+
880
+ # image_features: (batch_size, num_image, num_patch, hidden_size)
881
+ # cls_embed: (batch_size, num_image, hidden_size)
882
+ return image_features, cls_embed
883
+
884
+
885
+ def causal_attention_bias(seq_len: int, device: torch.device) -> torch.FloatTensor:
886
+ att_bias = torch.triu(
887
+ torch.ones(seq_len, seq_len, device=device, dtype=torch.float),
888
+ diagonal=1,
889
+ )
890
+ att_bias.masked_fill_(att_bias == 1, torch.finfo(att_bias.dtype).min)
891
+ return att_bias.view(1, 1, seq_len, seq_len) # type: ignore
892
+
893
+
894
+ class MolmoRmsLayerNorm(nn.Module):
895
+ """
896
+ RMS layer norm, a simplified :class:`LayerNorm` implementation
897
+ """
898
+
899
+ def __init__(
900
+ self,
901
+ config: MolmoConfig,
902
+ size: Optional[int] = None,
903
+ elementwise_affine: Optional[bool] = None,
904
+ eps: float = 1e-5,
905
+ ):
906
+ super().__init__()
907
+ self.config = config
908
+ self.eps = self.config.layer_norm_eps or eps
909
+ self.normalized_shape = (size or config.hidden_size,)
910
+ if elementwise_affine or (elementwise_affine is None):
911
+ self.weight = nn.Parameter(torch.ones(self.normalized_shape))
912
+ use_bias = self.config.bias_for_layer_norm
913
+ if use_bias:
914
+ self.bias = nn.Parameter(torch.zeros(self.normalized_shape))
915
+ else:
916
+ self.register_parameter("bias", None)
917
+ else:
918
+ self.register_parameter("bias", None)
919
+ self.register_parameter("weight", None)
920
+
921
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
922
+ with torch.autocast(enabled=False, device_type=x.device.type):
923
+ og_dtype = x.dtype
924
+ x = x.to(torch.float32)
925
+ variance = x.pow(2).mean(-1, keepdim=True)
926
+ x = x * torch.rsqrt(variance + self.eps)
927
+ x = x.to(og_dtype)
928
+
929
+ if self.weight is not None:
930
+ if self.bias is not None:
931
+ return self.weight * x + self.bias
932
+ else:
933
+ return self.weight * x
934
+ else:
935
+ return x
936
+
937
+
938
+ class MolmoModel(MolmoPreTrainedModel):
939
+ def __init__(self, config: MolmoConfig, init_params: bool = True):
940
+ super().__init__(config)
941
+
942
+ if self.config.additional_vocab_size is not None:
943
+ wte = Embedding(
944
+ config.vocab_size,
945
+ config.additional_vocab_size,
946
+ config.hidden_size,
947
+ )
948
+ else:
949
+ wte = nn.Embedding(config.vocab_size, config.hidden_size)
950
+
951
+ self.transformer = nn.ModuleDict(
952
+ dict(
953
+ wte=wte,
954
+ emb_drop=nn.Dropout(config.embedding_dropout),
955
+ ln_f=MolmoRmsLayerNorm(config),
956
+ )
957
+ )
958
+
959
+ if config.moe_num_experts > 0:
960
+ blocks = [MolmoeBlock(config) for i in range(config.num_hidden_layers)]
961
+ else:
962
+ blocks = [MolmoBlock(config) for i in range(config.num_hidden_layers)]
963
+ self.transformer.update({"blocks": nn.ModuleList(blocks)})
964
+
965
+ if not config.weight_tying:
966
+ self.transformer.update(
967
+ {
968
+ "ff_out": nn.Linear(
969
+ config.hidden_size,
970
+ config.vocab_size,
971
+ bias=False,
972
+ )
973
+ }
974
+ )
975
+
976
+ self.vision_backbone: Optional[OLMoVisionBackbone] = None
977
+ if config.vision_config is not None:
978
+ self.vision_backbone = OLMoVisionBackbone(config)
979
+
980
+ def reset_parameters(self):
981
+ if self.vision_backbone is not None:
982
+ self.vision_backbone.reset_parameters()
983
+ self.reset_non_vision_parameters()
984
+
985
+ def reset_non_vision_parameters(self):
986
+ self.transformer.wte.reset_parameters()
987
+ if hasattr(self.transformer.wte, "new_embedding"):
988
+ nn.init.normal_(self.transformer.wte.new_embedding, std=self.config.new_embedding_init_range)
989
+
990
+ if hasattr(self.transformer, "wpe"):
991
+ nn.init.normal_(self.transformer.wpe, mean=0.0, std=1.0)
992
+
993
+ self.transformer.ln_f.reset_parameters() # type: ignore
994
+
995
+ if hasattr(self.transformer, "ff_out"):
996
+ nn.init.normal_(self.transformer.ff_out, mean=0.0, std=0.02)
997
+
998
+ for block in self.transformer.blocks:
999
+ block.reset_parameters()
1000
+
1001
+ def forward(
1002
+ self,
1003
+ input_ids: torch.LongTensor,
1004
+ input_embeddings: Optional[torch.FloatTensor] = None,
1005
+ attention_mask: Optional[torch.Tensor] = None,
1006
+ images: Optional[torch.Tensor] = None,
1007
+ image_masks: Optional[torch.Tensor] = None,
1008
+ image_input_idx: Optional[torch.Tensor] = None,
1009
+ subsegment_ids: Optional[torch.Tensor] = None,
1010
+ position_ids: Optional[torch.Tensor] = None,
1011
+ past_key_values: Optional[Sequence[Tuple[torch.Tensor, torch.Tensor]]] = None,
1012
+ use_cache: bool = False,
1013
+ last_logits_only: bool = False,
1014
+ output_hidden_states: Optional[bool] = None,
1015
+ append_last_valid_logits: Optional[torch.Tensor] = None,
1016
+ ) -> ModelOutput:
1017
+ """
1018
+ :param input_ids: A tensor of shape `(batch_size, seq_len)`.
1019
+ :param input_embeddings: A tensor of shape `(batch_size, seq_len, hidden_size)` with input
1020
+ embeddings. When provided, it is treated as the output of the input embedding layer.
1021
+ :param attention_mask: A tensor of shape `(batch_size, seq_len)` that indicates
1022
+ which input IDs are masked. A `1` value in the mask means that
1023
+ the corresponding input ID should *not* be ignored. A `0` means
1024
+ that the corresponding input ID is masked.
1025
+
1026
+ This has the same meaning as the `attention_mask` in HuggingFace's `transformers`
1027
+ library.
1028
+ :param attention_bias: A tensor of shape `(batch_size, 1, seq_len, seq_len)`,
1029
+ `(1, 1, seq_len, seq_len)`, or `(seq_len, seq_len)`. This is used
1030
+ to introduce causal or other biases.
1031
+
1032
+ If the tensor is a bool or byte tensor, a `True` or `1` at `attention_bias[:, :, i, j]`
1033
+ indicates that the i-th element in the sequence is allowed to attend to the j-th
1034
+ element in the sequence.
1035
+
1036
+ If the tensor is a float tensor, it will just be added to the attention
1037
+ scores before the softmax.
1038
+
1039
+ The default is causal, which corresponds to a lower-diagonal byte matrix of ones.
1040
+ :param response_mask: A tensor of shape `(batch_size, seq_len)` that indicates
1041
+ the response mask. A `1` value in the mask means that the corresponding token
1042
+ is a response token. A `0` means that the corresponding token is not
1043
+ a response token.
1044
+ :param past_key_values: Pre-computed keys and values for each attention block.
1045
+ Can be used to speed up sequential decoding. The `input_ids` which have
1046
+ their past given to this model should not be passed as `input_ids` as they have already been computed.
1047
+ :param use_cache: If `True`, return key and value tensors for each block.
1048
+ :param last_logits_only: If `True`, only compute the logits for the last token of each sequence.
1049
+ This can speed up decoding when you only care about the next token.
1050
+ """
1051
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else False
1052
+
1053
+ if past_key_values:
1054
+ assert len(past_key_values) == self.config.num_hidden_layers
1055
+
1056
+ has_image = images is not None
1057
+
1058
+ assert not (has_image and input_embeddings is not None), "Cannot provide both images and input embeddings."
1059
+ assert not (has_image and past_key_values is not None), "Cached key and values should not be used with images."
1060
+
1061
+ batch_size, seq_len = input_ids.size() if input_embeddings is None else input_embeddings.size()[:2]
1062
+ if past_key_values is None:
1063
+ past_length = 0
1064
+ else:
1065
+ past_length = past_key_values[0][0].size(-2)
1066
+
1067
+ if attention_mask is None:
1068
+ attention_mask = input_ids != -1
1069
+
1070
+ if subsegment_ids is not None:
1071
+ raise NotImplementedError()
1072
+ else:
1073
+ if position_ids is None:
1074
+ position_ids = torch.clamp(
1075
+ torch.cumsum(attention_mask.to(torch.int32), dim=-1) - 1,
1076
+ min=0,
1077
+ ).broadcast_to((batch_size, attention_mask.shape[-1]))
1078
+
1079
+ # Get embeddings of input.
1080
+ # shape: (batch_size, seq_len, hidden_size)
1081
+ if input_ids is not None:
1082
+ input_ids = input_ids * (input_ids != -1).to(input_ids.dtype)
1083
+ x = self.transformer.wte(input_ids) if input_embeddings is None else input_embeddings # type: ignore
1084
+
1085
+ num_image: Optional[int] = None
1086
+ if images is not None:
1087
+ # shape: (batch_size, num_image, num_patch, hidden_size)
1088
+ # cls_embed: (batch_size, num_image, hidden_size)
1089
+ image_features, cls_embed = self.vision_backbone(images, image_masks)
1090
+ num_image, num_patch = image_features.shape[1:3]
1091
+ assert image_input_idx.shape == (batch_size, num_image, num_patch)
1092
+
1093
+ # inster the image feature into the embedding.
1094
+ image_features = image_features.view(batch_size, num_image * num_patch, -1)
1095
+ image_input_idx = image_input_idx.view(batch_size, num_image * num_patch)
1096
+
1097
+ valid = image_input_idx >= 0
1098
+ batch_idx = torch.arange(batch_size, device=x.device)
1099
+ batch_idx = torch.tile(batch_idx[:, None], [1, image_features.shape[1]])
1100
+
1101
+ # For hf demo/endpoint
1102
+ image_features = image_features.to(x.device)
1103
+
1104
+ x[batch_idx[valid], image_input_idx[valid]] += image_features[valid]
1105
+
1106
+ # Add input + positional embeddings and apply dropout.
1107
+ # shape: (batch_size, seq_len, hidden_size)
1108
+ x = self.transformer.emb_drop(x) # type: ignore
1109
+
1110
+ # normalized
1111
+ if self.config.normalize_input_embeds:
1112
+ x = x * (self.config.hidden_size ** 0.5)
1113
+
1114
+ # Merge attention mask with attention bias.
1115
+ # FIXME we are ignoring the attention mask input parameter
1116
+ if self.config.attention_type == "flash":
1117
+ attention_mask = input_ids != -1
1118
+ elif (
1119
+ attention_mask is not None
1120
+ or past_key_values is not None
1121
+ ):
1122
+ total_len = (past_length + seq_len)
1123
+ attention_mask = torch.tril(torch.ones(total_len, total_len, device=x.device, dtype=torch.bool))
1124
+ attention_mask = attention_mask.view(1, 1, total_len, total_len)
1125
+
1126
+ attn_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = [] if use_cache else None
1127
+
1128
+ # decoder layers
1129
+ all_hidden_states = []
1130
+
1131
+ # Apply blocks one-by-one.
1132
+ for block_idx, block in enumerate(self.transformer.blocks):
1133
+ if output_hidden_states:
1134
+ # add hidden states
1135
+ all_hidden_states.append(x)
1136
+
1137
+ layer_past = None if past_key_values is None else past_key_values[block_idx]
1138
+ x, cache = block(x, attention_mask=attention_mask, position_ids=position_ids, layer_past=layer_past, use_cache=use_cache)
1139
+
1140
+ if attn_key_values is not None:
1141
+ assert cache is not None
1142
+ attn_key_values.append(cache)
1143
+
1144
+ if last_logits_only:
1145
+ # shape: (batch_size, 1, hidden_size)
1146
+ if append_last_valid_logits is not None:
1147
+ last_valid_output = x[
1148
+ torch.arange(x.shape[0], device=x.device), append_last_valid_logits.to(x.device)]
1149
+ x = last_valid_output.unsqueeze(1)
1150
+ else:
1151
+ x = x[:, -1, :].unsqueeze(1)
1152
+
1153
+ # Apply final layer norm.
1154
+ # shape: (batch_size, seq_len or 1, hidden_size)
1155
+ x = self.transformer.ln_f(x) # type: ignore
1156
+ if output_hidden_states:
1157
+ # add final hidden state post-final-layernorm, following HuggingFace's convention
1158
+ all_hidden_states.append(x)
1159
+
1160
+ # Get logits.
1161
+ # shape: (batch_size, seq_len or 1, vocab_size)
1162
+ if self.config.weight_tying:
1163
+ logits = F.linear(x, self.transformer.wte.weight, None) # type: ignore
1164
+ else:
1165
+ logits = self.transformer.ff_out(x) # type: ignore
1166
+ if self.config.scale_logits:
1167
+ logits.mul_(1 / math.sqrt(self.config.hidden_size))
1168
+
1169
+ if not last_logits_only and append_last_valid_logits is not None:
1170
+ last_valid_logit = logits[
1171
+ torch.arange(logits.shape[0], device=logits.device), append_last_valid_logits]
1172
+ logits = torch.cat([logits[:, :-1], last_valid_logit[:, None]], dim=1)
1173
+
1174
+ return ModelOutput(logits=logits, attn_key_values=attn_key_values, hidden_states=tuple(all_hidden_states) if output_hidden_states else None) # type: ignore[arg-type]
1175
+
1176
+
1177
+ class MolmoForCausalLM(MolmoPreTrainedModel):
1178
+
1179
+ def __init__(self, config: MolmoConfig, model: Optional[MolmoModel] = None, init_params: bool = False):
1180
+ super().__init__(config)
1181
+
1182
+ if not model:
1183
+ self.model = MolmoModel(config, init_params=init_params)
1184
+ else:
1185
+ self.model = model
1186
+ self.post_init()
1187
+
1188
+ def get_input_embeddings(self) -> torch.nn.Module:
1189
+ return self.model.transformer.wte
1190
+
1191
+ def get_output_embeddings(self):
1192
+ if self.config.weight_tying:
1193
+ return self.model.transformer.wte
1194
+ else:
1195
+ return self.model.transformer.ff_out
1196
+
1197
+ def forward(
1198
+ self,
1199
+ input_ids: torch.LongTensor = None,
1200
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1201
+ attention_mask: Optional[torch.Tensor] = None,
1202
+ attention_bias: Optional[torch.Tensor] = None,
1203
+ response_mask: Optional[torch.Tensor] = None,
1204
+ images: Optional[torch.Tensor] = None,
1205
+ image_masks: Optional[torch.Tensor] = None,
1206
+ image_input_idx: Optional[torch.Tensor] = None,
1207
+ subsegment_ids: Optional[torch.Tensor] = None,
1208
+ position_ids: Optional[torch.Tensor] = None,
1209
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
1210
+ labels: Optional[torch.LongTensor] = None,
1211
+ loss_masks: Optional[torch.Tensor] = None,
1212
+ use_cache: Optional[bool] = None,
1213
+ last_logits_only: Optional[bool] = None,
1214
+ output_attentions: Optional[bool] = None,
1215
+ output_hidden_states: Optional[bool] = None,
1216
+ append_last_valid_logits: Optional[torch.Tensor] = None,
1217
+ return_dict: Optional[bool] = None,
1218
+ cache_position: Optional[
1219
+ Cache
1220
+ ] = None, # This is a hack mitigation of an issue in transformers `4.39.x` https://github.com/huggingface/transformers/issues/29426
1221
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1222
+ if use_cache is None:
1223
+ use_cache = self.config.use_cache
1224
+
1225
+ if output_attentions:
1226
+ raise ValueError("output_attentions is not yet supported in Molmo")
1227
+
1228
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1229
+
1230
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
1231
+ outputs = self.model.forward(
1232
+ input_ids=input_ids,
1233
+ input_embeddings=inputs_embeds,
1234
+ attention_mask=attention_mask,
1235
+ images=images,
1236
+ image_masks=image_masks,
1237
+ image_input_idx=image_input_idx,
1238
+ subsegment_ids=subsegment_ids,
1239
+ position_ids=position_ids,
1240
+ past_key_values=past_key_values,
1241
+ use_cache=use_cache,
1242
+ last_logits_only=last_logits_only,
1243
+ output_hidden_states=output_hidden_states,
1244
+ append_last_valid_logits=append_last_valid_logits,
1245
+ )
1246
+
1247
+ logits = outputs.logits
1248
+ hidden_states = outputs.hidden_states
1249
+
1250
+ loss = None
1251
+ if labels is not None:
1252
+ if loss_masks is not None:
1253
+ loss_masks = loss_masks * (loss_masks > 0)
1254
+ batch_size_in_tokens = max(loss_masks.sum().item(), 1)
1255
+ labels = labels.long()
1256
+ labels.masked_fill_(~(loss_masks > 0), -100)
1257
+ labels = labels.view(-1)
1258
+ logits_for_loss = logits.to(torch.float32).view(-1, logits.size(-1))
1259
+ loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100, reduction='none')
1260
+ loss = loss_fct(logits_for_loss, labels)
1261
+ loss = loss.view(input_ids.shape[0], -1)
1262
+ loss = loss * loss_masks
1263
+ loss = loss.sum() / batch_size_in_tokens
1264
+ use_zloss = getattr(self.config, "softmax_auxiliary_loss", False)
1265
+ if use_zloss:
1266
+ z_squared = logits_for_loss.logsumexp(-1).pow(2)
1267
+ z_loss = self.config.softmax_auxiliary_loss_scale * z_squared
1268
+ z_loss = z_loss.view(input_ids.shape[0], -1)
1269
+ z_loss = z_loss * loss_masks
1270
+ z_loss = z_loss.sum() / batch_size_in_tokens
1271
+ loss += z_loss
1272
+ else:
1273
+ # Shift so that tokens < n predict n
1274
+ shift_logits = logits[..., :-1, :].contiguous()
1275
+ shift_labels = labels[..., 1:].contiguous()
1276
+ # Flatten the tokens
1277
+ loss_fct = torch.nn.CrossEntropyLoss()
1278
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
1279
+ shift_labels = shift_labels.view(-1)
1280
+ # Enable model parallelism
1281
+ shift_labels = shift_labels.to(shift_logits.device)
1282
+ loss = loss_fct(shift_logits, shift_labels)
1283
+
1284
+ if not return_dict:
1285
+ output = (logits,) + outputs[1:]
1286
+ return (loss,) + output if loss is not None else output
1287
+
1288
+ return CausalLMOutputWithPast(
1289
+ loss=loss,
1290
+ logits=logits,
1291
+ past_key_values=outputs.attn_key_values,
1292
+ hidden_states=hidden_states,
1293
+ )
1294
+
1295
+ def can_generate(self) -> bool:
1296
+ return True
1297
+
1298
+ @torch.no_grad()
1299
+ def generate_from_batch(
1300
+ self,
1301
+ batch: Dict[str, Any],
1302
+ generation_config: Optional[GenerationConfig] = None,
1303
+ **kwargs,
1304
+ ):
1305
+ if generation_config is not None:
1306
+ assert generation_config.use_cache
1307
+
1308
+ images = batch.get("images")
1309
+ image_masks = batch.get("image_masks")
1310
+ image_input_idx = batch.get("image_input_idx")
1311
+
1312
+ # Validate inputs.
1313
+ input_ids = batch["input_ids"]
1314
+ batch_size, seq_len = input_ids.shape
1315
+ attention_mask = batch.get("attention_mask", None)
1316
+ max_new_tokens = generation_config.max_new_tokens
1317
+ assert max_new_tokens is not None
1318
+ mask_len = seq_len + max_new_tokens
1319
+ position_ids: Optional[torch.Tensor] = None
1320
+ append_last_valid_logits: Optional[torch.Tensor] = None
1321
+ if attention_mask is None:
1322
+ attention_mask = input_ids != -1
1323
+ position_ids = torch.clamp(
1324
+ torch.cumsum(attention_mask.to(torch.int32), dim=-1) - 1,
1325
+ min=0
1326
+ )
1327
+ append_last_valid_logits = attention_mask.long().sum(dim=-1) - 1
1328
+ attention_mask = torch.cat(
1329
+ [attention_mask, attention_mask.new_ones((batch_size, max_new_tokens))],
1330
+ dim=1,
1331
+ )
1332
+ if attention_mask is not None:
1333
+ assert attention_mask.shape == (batch_size, mask_len)
1334
+
1335
+ out = super().generate(
1336
+ batch["input_ids"],
1337
+ generation_config,
1338
+ attention_mask=attention_mask,
1339
+ images=images,
1340
+ image_masks=image_masks,
1341
+ image_input_idx=image_input_idx,
1342
+ position_ids=position_ids,
1343
+ append_last_valid_logits=append_last_valid_logits,
1344
+ **kwargs,
1345
+ )
1346
+
1347
+ return out
1348
+
1349
+ def prepare_inputs_for_generation(
1350
+ self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple]] = None, **kwargs
1351
+ ):
1352
+ if past_key_values:
1353
+ # This is because we want the model to only process the last generated token.
1354
+ input_ids = input_ids[:, -1:]
1355
+
1356
+ attention_mask = kwargs.get("attention_mask")
1357
+ images = kwargs.get("images")
1358
+ image_masks = kwargs.get("image_masks")
1359
+ image_input_idx = kwargs.get("image_input_idx")
1360
+ position_ids = kwargs.get("position_ids")
1361
+ append_last_valid_logits = kwargs.get("append_last_valid_logits")
1362
+ model_inputs = {
1363
+ "input_ids": input_ids,
1364
+ "attention_mask": attention_mask,
1365
+ "position_ids": position_ids,
1366
+ "past_key_values": past_key_values,
1367
+ "use_cache": True,
1368
+ "last_logits_only": True,
1369
+ }
1370
+ if past_key_values is None:
1371
+ model_inputs["images"] = images
1372
+ model_inputs["image_masks"] = image_masks
1373
+ model_inputs["image_input_idx"] = image_input_idx
1374
+ model_inputs["append_last_valid_logits"] = append_last_valid_logits
1375
+ return model_inputs
1376
+
1377
+ def _update_model_kwargs_for_generation(
1378
+ self,
1379
+ outputs: ModelOutput,
1380
+ model_kwargs: Dict[str, Any],
1381
+ is_encoder_decoder: bool = False,
1382
+ num_new_tokens: int = 1,
1383
+ ) -> Dict[str, Any]:
1384
+ model_kwargs["position_ids"] = model_kwargs["position_ids"][:, -1:] + 1
1385
+ if "append_last_valid_logits" in model_kwargs:
1386
+ del model_kwargs["append_last_valid_logits"]
1387
+ if "images" in model_kwargs:
1388
+ del model_kwargs["images"]
1389
+ del model_kwargs["image_masks"]
1390
+ del model_kwargs["image_input_idx"]
1391
+ # https://huggingface.co/allenai/Molmo-7B-D-0924/blob/main/modeling_molmo.py#L2275
1392
+ try:
1393
+ cache_name, cache = super()._extract_past_from_model_output(outputs)
1394
+ except:
1395
+ past_key_values = None
1396
+ if "past_key_values" in outputs:
1397
+ past_key_values = outputs.past_key_values
1398
+ # elif "mems" in outputs:
1399
+ # past_key_values = outputs.mems
1400
+ # elif "past_buckets_states" in outputs:
1401
+ # past_key_values = outputs.past_buckets_states
1402
+ cache_name, cache = "past_key_values", past_key_values
1403
+ model_kwargs[cache_name] = cache
1404
+ model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens
1405
+ return model_kwargs
1406
+
1407
+
1408
+ # Always register for multi-modal features
1409
+ AutoModelForCausalLM.register(MolmoConfig, MolmoForCausalLM)
preprocessing_molmo.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Processor class for Molmo.
3
+ """
4
+
5
+ from typing import Optional
6
+
7
+ import PIL
8
+ from PIL import ImageOps
9
+ from PIL.Image import Image
10
+
11
+ try:
12
+ from typing import Unpack
13
+ except ImportError:
14
+ from typing_extensions import Unpack
15
+
16
+ import numpy as np
17
+ import torch
18
+
19
+ from transformers.image_utils import ImageInput
20
+ from transformers.processing_utils import (
21
+ TextKwargs,
22
+ ProcessingKwargs,
23
+ ProcessorMixin,
24
+ )
25
+
26
+ from transformers.tokenization_utils_base import TextInput
27
+ from transformers.utils import logging
28
+
29
+ from transformers import AutoTokenizer
30
+ from .image_preprocessing_molmo import MolmoImagesKwargs, MolmoImageProcessor
31
+
32
+
33
+ logger = logging.get_logger(__name__)
34
+
35
+
36
+ DEFAULT_IMAGE_PATCH_TOKEN = f"<im_patch>"
37
+ DEFAULT_IM_START_TOKEN = f"<im_start>"
38
+ DEFAULT_IM_END_TOKEN = f"<im_end>"
39
+ DEFAULT_IM_COL_TOKEN = f"<im_col>"
40
+ IMAGE_PROMPT = "<|image|>"
41
+
42
+ EXTRA_TOKENS = (DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_COL_TOKEN, IMAGE_PROMPT)
43
+
44
+
45
+ def get_special_token_ids(tokenizer):
46
+ ids = tokenizer.encode("".join(EXTRA_TOKENS), add_special_tokens=False)
47
+ assert len(ids) == len(EXTRA_TOKENS)
48
+ return {k: i for k, i in zip(EXTRA_TOKENS, ids)}
49
+
50
+
51
+ class MolmoTextKwargs(TextKwargs, total=False):
52
+ style: Optional[str]
53
+ system_prompt: Optional[str]
54
+ message_format: Optional[str]
55
+ always_start_with_space: Optional[bool]
56
+ sequence_length: Optional[int]
57
+
58
+
59
+ class MolmoProcessorKwargs(ProcessingKwargs, total=False):
60
+ text_kwargs: MolmoTextKwargs
61
+ images_kwargs: MolmoImagesKwargs
62
+ _defaults = {
63
+ "images_kwargs": {
64
+ "max_crops": 12,
65
+ "overlap_margins": [4, 4],
66
+ "base_image_input_size": [336, 336],
67
+ "image_token_length_w": 12,
68
+ "image_token_length_h": 12,
69
+ "image_patch_size": 14,
70
+ "image_padding_mask": True,
71
+ },
72
+ "text_kwargs": {
73
+ "style": "long_caption",
74
+ "system_prompt": "none",
75
+ "message_format": "role",
76
+ "always_start_with_space": True,
77
+ "sequence_length": 1536,
78
+ "padding": False,
79
+ },
80
+ }
81
+
82
+
83
+ class MolmoProcessor(ProcessorMixin):
84
+ attributes = ["image_processor", "tokenizer"]
85
+ image_processor_class = "AutoImageProcessor"
86
+ tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
87
+
88
+ def __init__(self, image_processor: MolmoImageProcessor = None, tokenizer : AutoTokenizer = None, **kwargs):
89
+ # self.image_processor = image_processor
90
+ # self.tokenizer = tokenizer
91
+ super().__init__(image_processor, tokenizer)
92
+ self._special_tokens = None
93
+
94
+ @property
95
+ def special_token_ids(self):
96
+ if self._special_tokens is None:
97
+ self._special_tokens = get_special_token_ids(self.tokenizer)
98
+ return self._special_tokens
99
+
100
+ def get_tokens_input(self, prompt, message_format, always_start_with_space):
101
+ if message_format == "none" or message_format is None:
102
+ pass
103
+ elif message_format == "role":
104
+ prompt = "User: " + prompt + " Assistant:"
105
+ else:
106
+ raise NotImplementedError(f"Message format {message_format} not implemented")
107
+
108
+ if always_start_with_space:
109
+ prompt = " " + prompt
110
+
111
+ tokens = self.tokenizer.encode(prompt, add_special_tokens=False)
112
+
113
+ return tokens
114
+
115
+ def process(
116
+ self,
117
+ text: TextInput = None,
118
+ images: ImageInput = None,
119
+ **kwargs: Unpack[MolmoProcessorKwargs],
120
+ ):
121
+ output_kwargs = self._merge_kwargs(
122
+ MolmoProcessorKwargs,
123
+ tokenizer_init_kwargs=self.tokenizer.init_kwargs,
124
+ **kwargs,
125
+ )
126
+
127
+ tokens = self.get_tokens_input(
128
+ text,
129
+ output_kwargs["text_kwargs"]["message_format"],
130
+ output_kwargs["text_kwargs"]["always_start_with_space"],
131
+ )
132
+
133
+ image_token_id = self.special_token_ids[IMAGE_PROMPT]
134
+
135
+ if images is not None:
136
+ if not isinstance(images, (list, tuple)):
137
+ images = [images]
138
+ image_arrays = []
139
+ for image in images:
140
+ if isinstance(image, Image):
141
+ image = image.convert("RGB")
142
+ # Handle images with EXIF orientation tags, which PIL will ignore by default
143
+ # https://github.com/python-pillow/Pillow/issues/4703
144
+ img = ImageOps.exif_transpose(image)
145
+ image_arrays.append(np.array(image))
146
+ else:
147
+ assert len(image.shape) == 3 and image.shape[-1] == 3
148
+ image_arrays.append(image.astype(np.uint8))
149
+ images = image_arrays
150
+ # For now only support inserting images at the start
151
+ image_idx = [-1]*len(images)
152
+ else:
153
+ image_idx = None
154
+
155
+ sequence_length = output_kwargs["text_kwargs"]["sequence_length"]
156
+
157
+ image_patch_token_id = self.special_token_ids[DEFAULT_IMAGE_PATCH_TOKEN]
158
+ image_col_token_id = self.special_token_ids[DEFAULT_IM_COL_TOKEN]
159
+ image_start_token_id = self.special_token_ids[DEFAULT_IM_START_TOKEN]
160
+ image_end_token_id = self.special_token_ids[DEFAULT_IM_END_TOKEN]
161
+ out = self.image_processor.multimodal_preprocess(
162
+ images=images,
163
+ image_idx=image_idx,
164
+ tokens=np.asarray(tokens).astype(np.int32),
165
+ sequence_length=sequence_length,
166
+ image_patch_token_id=image_patch_token_id,
167
+ image_col_token_id=image_col_token_id,
168
+ image_start_token_id=image_start_token_id,
169
+ image_end_token_id=image_end_token_id,
170
+ **output_kwargs["images_kwargs"]
171
+ )
172
+
173
+ # Prepend BOS
174
+ # qwen2 and olmo do not have a BOS, and instead use EOS as a generic seperator token.
175
+ bos = self.tokenizer.bos_token_id or self.tokenizer.eos_token_id
176
+ decoder_input_tokens = np.pad(out["input_ids"], [[1, 0]], constant_values=bos)
177
+ out["input_ids"] = decoder_input_tokens
178
+ if "image_input_idx" in out:
179
+ # Shift patch mapping up by one since we added BOS
180
+ image_input_idx = out["image_input_idx"]
181
+ out["image_input_idx"] = np.where(image_input_idx < 0, image_input_idx, image_input_idx + 1)
182
+
183
+ for k, v in out.items():
184
+ out[k] = torch.from_numpy(v)
185
+
186
+ return out
187
+
188
+
189
+ MolmoProcessor.register_for_auto_class()
preprocessor_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoImageProcessor": "image_preprocessing_molmo.MolmoImageProcessor",
4
+ "AutoProcessor": "preprocessing_molmo.MolmoProcessor"
5
+ },
6
+ "base_image_input_size": [
7
+ 336,
8
+ 336
9
+ ],
10
+ "do_normalize": true,
11
+ "image_padding_mask": true,
12
+ "image_patch_size": 14,
13
+ "image_processor_type": "MolmoImageProcessor",
14
+ "image_token_length_h": 12,
15
+ "image_token_length_w": 12,
16
+ "max_crops": 12,
17
+ "overlap_margins": [
18
+ 4,
19
+ 4
20
+ ],
21
+ "processor_class": "MolmoProcessor"
22
+ }
processor_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoProcessor": "preprocessing_molmo.MolmoProcessor"
4
+ },
5
+ "processor_class": "MolmoProcessor"
6
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,441 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "|<EXTRA_TOKENS_0>|",
4
+ "|<EXTRA_TOKENS_1>|",
5
+ "|<EXTRA_TOKENS_2>|",
6
+ "|<EXTRA_TOKENS_3>|",
7
+ "|<EXTRA_TOKENS_4>|",
8
+ "|<EXTRA_TOKENS_5>|",
9
+ "|<EXTRA_TOKENS_6>|",
10
+ "|<EXTRA_TOKENS_7>|",
11
+ "|<EXTRA_TOKENS_8>|",
12
+ "|<EXTRA_TOKENS_9>|",
13
+ "|<EXTRA_TOKENS_10>|",
14
+ "|<EXTRA_TOKENS_11>|",
15
+ "|<EXTRA_TOKENS_12>|",
16
+ "|<EXTRA_TOKENS_13>|",
17
+ "|<EXTRA_TOKENS_14>|",
18
+ "|<EXTRA_TOKENS_15>|",
19
+ "|<EXTRA_TOKENS_16>|",
20
+ "|<EXTRA_TOKENS_17>|",
21
+ "|<EXTRA_TOKENS_18>|",
22
+ "|<EXTRA_TOKENS_19>|",
23
+ "|<EXTRA_TOKENS_20>|",
24
+ "|<EXTRA_TOKENS_21>|",
25
+ "|<EXTRA_TOKENS_22>|",
26
+ "|<EXTRA_TOKENS_23>|",
27
+ "|<EXTRA_TOKENS_24>|",
28
+ "|<EXTRA_TOKENS_25>|",
29
+ "|<EXTRA_TOKENS_26>|",
30
+ "|<EXTRA_TOKENS_27>|",
31
+ "|<EXTRA_TOKENS_28>|",
32
+ "|<EXTRA_TOKENS_29>|",
33
+ "|<EXTRA_TOKENS_30>|",
34
+ "|<EXTRA_TOKENS_31>|",
35
+ "|<EXTRA_TOKENS_32>|",
36
+ "|<EXTRA_TOKENS_33>|",
37
+ "|<EXTRA_TOKENS_34>|",
38
+ "|<EXTRA_TOKENS_35>|",
39
+ "|<EXTRA_TOKENS_36>|",
40
+ "|<EXTRA_TOKENS_37>|",
41
+ "|<EXTRA_TOKENS_38>|",
42
+ "|<EXTRA_TOKENS_39>|",
43
+ "|<EXTRA_TOKENS_40>|",
44
+ "|<EXTRA_TOKENS_41>|",
45
+ "|<EXTRA_TOKENS_42>|",
46
+ "|<EXTRA_TOKENS_43>|",
47
+ "|<EXTRA_TOKENS_44>|",
48
+ "|<EXTRA_TOKENS_45>|",
49
+ "|<EXTRA_TOKENS_46>|",
50
+ "|<EXTRA_TOKENS_47>|",
51
+ "|<EXTRA_TOKENS_48>|",
52
+ "|<EXTRA_TOKENS_49>|",
53
+ "|<EXTRA_TOKENS_50>|",
54
+ "|<EXTRA_TOKENS_51>|",
55
+ "|<EXTRA_TOKENS_52>|",
56
+ "|<EXTRA_TOKENS_53>|",
57
+ "|<EXTRA_TOKENS_54>|",
58
+ "|<EXTRA_TOKENS_55>|",
59
+ "|<EXTRA_TOKENS_56>|",
60
+ "|<EXTRA_TOKENS_57>|",
61
+ "|<EXTRA_TOKENS_58>|",
62
+ "|<EXTRA_TOKENS_59>|",
63
+ "|<EXTRA_TOKENS_60>|",
64
+ "|<EXTRA_TOKENS_61>|",
65
+ "|<EXTRA_TOKENS_62>|",
66
+ "|<EXTRA_TOKENS_63>|",
67
+ "|<EXTRA_TOKENS_64>|",
68
+ "|<EXTRA_TOKENS_65>|",
69
+ "|<EXTRA_TOKENS_66>|",
70
+ "|<EXTRA_TOKENS_67>|",
71
+ "|<EXTRA_TOKENS_68>|",
72
+ "|<EXTRA_TOKENS_69>|",
73
+ "|<EXTRA_TOKENS_70>|",
74
+ "|<EXTRA_TOKENS_71>|",
75
+ "|<EXTRA_TOKENS_72>|",
76
+ "|<EXTRA_TOKENS_73>|",
77
+ "|<EXTRA_TOKENS_74>|",
78
+ "|<EXTRA_TOKENS_75>|",
79
+ "|<EXTRA_TOKENS_76>|",
80
+ "|<EXTRA_TOKENS_77>|",
81
+ "|<EXTRA_TOKENS_78>|",
82
+ "|<EXTRA_TOKENS_79>|",
83
+ "|<EXTRA_TOKENS_80>|",
84
+ "|<EXTRA_TOKENS_81>|",
85
+ "|<EXTRA_TOKENS_82>|",
86
+ "|<EXTRA_TOKENS_83>|",
87
+ "|<EXTRA_TOKENS_84>|",
88
+ "|<EXTRA_TOKENS_85>|",
89
+ "|<EXTRA_TOKENS_86>|",
90
+ "|<EXTRA_TOKENS_87>|",
91
+ "|<EXTRA_TOKENS_88>|",
92
+ "|<EXTRA_TOKENS_89>|",
93
+ "|<EXTRA_TOKENS_90>|",
94
+ "|<EXTRA_TOKENS_91>|",
95
+ "|<EXTRA_TOKENS_92>|",
96
+ "|<EXTRA_TOKENS_93>|",
97
+ "|<EXTRA_TOKENS_94>|",
98
+ "|<EXTRA_TOKENS_95>|",
99
+ "|<EXTRA_TOKENS_96>|",
100
+ "|<EXTRA_TOKENS_97>|",
101
+ "|<EXTRA_TOKENS_98>|",
102
+ "|<EXTRA_TOKENS_99>|",
103
+ "|<EXTRA_TOKENS_100>|",
104
+ "|<EXTRA_TOKENS_101>|",
105
+ "|<EXTRA_TOKENS_102>|",
106
+ "|<EXTRA_TOKENS_103>|",
107
+ "|<EXTRA_TOKENS_104>|",
108
+ "|<EXTRA_TOKENS_105>|",
109
+ "|<EXTRA_TOKENS_106>|",
110
+ "|<EXTRA_TOKENS_107>|",
111
+ "|<EXTRA_TOKENS_108>|",
112
+ "|<EXTRA_TOKENS_109>|",
113
+ "|<EXTRA_TOKENS_110>|",
114
+ "|<EXTRA_TOKENS_111>|",
115
+ "|<EXTRA_TOKENS_112>|",
116
+ "|<EXTRA_TOKENS_113>|",
117
+ "|<EXTRA_TOKENS_114>|",
118
+ "|<EXTRA_TOKENS_115>|",
119
+ "|<EXTRA_TOKENS_116>|",
120
+ "|<EXTRA_TOKENS_117>|",
121
+ "|<EXTRA_TOKENS_118>|",
122
+ "|<EXTRA_TOKENS_119>|",
123
+ "|<EXTRA_TOKENS_120>|",
124
+ "|<EXTRA_TOKENS_121>|",
125
+ "|<EXTRA_TOKENS_122>|",
126
+ "|<EXTRA_TOKENS_123>|",
127
+ "|<EXTRA_TOKENS_124>|",
128
+ "|<EXTRA_TOKENS_125>|",
129
+ "|<EXTRA_TOKENS_126>|",
130
+ "|<EXTRA_TOKENS_127>|",
131
+ "|<EXTRA_TOKENS_128>|",
132
+ "|<EXTRA_TOKENS_129>|",
133
+ "|<EXTRA_TOKENS_130>|",
134
+ "|<EXTRA_TOKENS_131>|",
135
+ "|<EXTRA_TOKENS_132>|",
136
+ "|<EXTRA_TOKENS_133>|",
137
+ "|<EXTRA_TOKENS_134>|",
138
+ "|<EXTRA_TOKENS_135>|",
139
+ "|<EXTRA_TOKENS_136>|",
140
+ "|<EXTRA_TOKENS_137>|",
141
+ "|<EXTRA_TOKENS_138>|",
142
+ "|<EXTRA_TOKENS_139>|",
143
+ "|<EXTRA_TOKENS_140>|",
144
+ "|<EXTRA_TOKENS_141>|",
145
+ "|<EXTRA_TOKENS_142>|",
146
+ "|<EXTRA_TOKENS_143>|",
147
+ "|<EXTRA_TOKENS_144>|",
148
+ "|<EXTRA_TOKENS_145>|",
149
+ "|<EXTRA_TOKENS_146>|",
150
+ "|<EXTRA_TOKENS_147>|",
151
+ "|<EXTRA_TOKENS_148>|",
152
+ "|<EXTRA_TOKENS_149>|",
153
+ "|<EXTRA_TOKENS_150>|",
154
+ "|<EXTRA_TOKENS_151>|",
155
+ "|<EXTRA_TOKENS_152>|",
156
+ "|<EXTRA_TOKENS_153>|",
157
+ "|<EXTRA_TOKENS_154>|",
158
+ "|<EXTRA_TOKENS_155>|",
159
+ "|<EXTRA_TOKENS_156>|",
160
+ "|<EXTRA_TOKENS_157>|",
161
+ "|<EXTRA_TOKENS_158>|",
162
+ "|<EXTRA_TOKENS_159>|",
163
+ "|<EXTRA_TOKENS_160>|",
164
+ "|<EXTRA_TOKENS_161>|",
165
+ "|<EXTRA_TOKENS_162>|",
166
+ "|<EXTRA_TOKENS_163>|",
167
+ "|<EXTRA_TOKENS_164>|",
168
+ "|<EXTRA_TOKENS_165>|",
169
+ "|<EXTRA_TOKENS_166>|",
170
+ "|<EXTRA_TOKENS_167>|",
171
+ "|<EXTRA_TOKENS_168>|",
172
+ "|<EXTRA_TOKENS_169>|",
173
+ "|<EXTRA_TOKENS_170>|",
174
+ "|<EXTRA_TOKENS_171>|",
175
+ "|<EXTRA_TOKENS_172>|",
176
+ "|<EXTRA_TOKENS_173>|",
177
+ "|<EXTRA_TOKENS_174>|",
178
+ "|<EXTRA_TOKENS_175>|",
179
+ "|<EXTRA_TOKENS_176>|",
180
+ "|<EXTRA_TOKENS_177>|",
181
+ "|<EXTRA_TOKENS_178>|",
182
+ "|<EXTRA_TOKENS_179>|",
183
+ "|<EXTRA_TOKENS_180>|",
184
+ "|<EXTRA_TOKENS_181>|",
185
+ "|<EXTRA_TOKENS_182>|",
186
+ "|<EXTRA_TOKENS_183>|",
187
+ "|<EXTRA_TOKENS_184>|",
188
+ "|<EXTRA_TOKENS_185>|",
189
+ "|<EXTRA_TOKENS_186>|",
190
+ "|<EXTRA_TOKENS_187>|",
191
+ "|<EXTRA_TOKENS_188>|",
192
+ "|<EXTRA_TOKENS_189>|",
193
+ "|<EXTRA_TOKENS_190>|",
194
+ "|<EXTRA_TOKENS_191>|",
195
+ "|<EXTRA_TOKENS_192>|",
196
+ "|<EXTRA_TOKENS_193>|",
197
+ "|<EXTRA_TOKENS_194>|",
198
+ "|<EXTRA_TOKENS_195>|",
199
+ "|<EXTRA_TOKENS_196>|",
200
+ "|<EXTRA_TOKENS_197>|",
201
+ "|<EXTRA_TOKENS_198>|",
202
+ "|<EXTRA_TOKENS_199>|",
203
+ "|<EXTRA_TOKENS_200>|",
204
+ "|<EXTRA_TOKENS_201>|",
205
+ "|<EXTRA_TOKENS_202>|",
206
+ "|<EXTRA_TOKENS_203>|",
207
+ "|<EXTRA_TOKENS_204>|",
208
+ "|<EXTRA_TOKENS_205>|",
209
+ "|<EXTRA_TOKENS_206>|",
210
+ "|<EXTRA_TOKENS_207>|",
211
+ "|<EXTRA_TOKENS_208>|",
212
+ "|<EXTRA_TOKENS_209>|",
213
+ "|<EXTRA_TOKENS_210>|",
214
+ "|<EXTRA_TOKENS_211>|",
215
+ "|<EXTRA_TOKENS_212>|",
216
+ "|<EXTRA_TOKENS_213>|",
217
+ "|<EXTRA_TOKENS_214>|",
218
+ "|<EXTRA_TOKENS_215>|",
219
+ "|<EXTRA_TOKENS_216>|",
220
+ "|<EXTRA_TOKENS_217>|",
221
+ "|<EXTRA_TOKENS_218>|",
222
+ "|<EXTRA_TOKENS_219>|",
223
+ "|<EXTRA_TOKENS_220>|",
224
+ "|<EXTRA_TOKENS_221>|",
225
+ "|<EXTRA_TOKENS_222>|",
226
+ "|<EXTRA_TOKENS_223>|",
227
+ "|<EXTRA_TOKENS_224>|",
228
+ "|<EXTRA_TOKENS_225>|",
229
+ "|<EXTRA_TOKENS_226>|",
230
+ "|<EXTRA_TOKENS_227>|",
231
+ "|<EXTRA_TOKENS_228>|",
232
+ "|<EXTRA_TOKENS_229>|",
233
+ "|<EXTRA_TOKENS_230>|",
234
+ "|<EXTRA_TOKENS_231>|",
235
+ "|<EXTRA_TOKENS_232>|",
236
+ "|<EXTRA_TOKENS_233>|",
237
+ "|<EXTRA_TOKENS_234>|",
238
+ "|<EXTRA_TOKENS_235>|",
239
+ "|<EXTRA_TOKENS_236>|",
240
+ "|<EXTRA_TOKENS_237>|",
241
+ "|<EXTRA_TOKENS_238>|",
242
+ "|<EXTRA_TOKENS_239>|",
243
+ "|<EXTRA_TOKENS_240>|",
244
+ "|<EXTRA_TOKENS_241>|",
245
+ "|<EXTRA_TOKENS_242>|",
246
+ "|<EXTRA_TOKENS_243>|",
247
+ "|<EXTRA_TOKENS_244>|",
248
+ "|<EXTRA_TOKENS_245>|",
249
+ "|<EXTRA_TOKENS_246>|",
250
+ "|<EXTRA_TOKENS_247>|",
251
+ "|<EXTRA_TOKENS_248>|",
252
+ "|<EXTRA_TOKENS_249>|",
253
+ "|<EXTRA_TOKENS_250>|",
254
+ "|<EXTRA_TOKENS_251>|",
255
+ "|<EXTRA_TOKENS_252>|",
256
+ "|<EXTRA_TOKENS_253>|",
257
+ "|<EXTRA_TOKENS_254>|",
258
+ "|<EXTRA_TOKENS_255>|",
259
+ "|<EXTRA_TOKENS_256>|",
260
+ "|<EXTRA_TOKENS_257>|",
261
+ "|<EXTRA_TOKENS_258>|",
262
+ "|<EXTRA_TOKENS_259>|",
263
+ "|<EXTRA_TOKENS_260>|",
264
+ "|<EXTRA_TOKENS_261>|",
265
+ "|<EXTRA_TOKENS_262>|",
266
+ "|<EXTRA_TOKENS_263>|",
267
+ "|<EXTRA_TOKENS_264>|",
268
+ "|<EXTRA_TOKENS_265>|",
269
+ "|<EXTRA_TOKENS_266>|",
270
+ "|<EXTRA_TOKENS_267>|",
271
+ "|<EXTRA_TOKENS_268>|",
272
+ "|<EXTRA_TOKENS_269>|",
273
+ "|<EXTRA_TOKENS_270>|",
274
+ "|<EXTRA_TOKENS_271>|",
275
+ "|<EXTRA_TOKENS_272>|",
276
+ "|<EXTRA_TOKENS_273>|",
277
+ "|<EXTRA_TOKENS_274>|",
278
+ "|<EXTRA_TOKENS_275>|",
279
+ "|<EXTRA_TOKENS_276>|",
280
+ "|<EXTRA_TOKENS_277>|",
281
+ "|<EXTRA_TOKENS_278>|",
282
+ "|<EXTRA_TOKENS_279>|",
283
+ "|<EXTRA_TOKENS_280>|",
284
+ "|<EXTRA_TOKENS_281>|",
285
+ "|<EXTRA_TOKENS_282>|",
286
+ "|<EXTRA_TOKENS_283>|",
287
+ "|<EXTRA_TOKENS_284>|",
288
+ "|<EXTRA_TOKENS_285>|",
289
+ "|<EXTRA_TOKENS_286>|",
290
+ "|<EXTRA_TOKENS_287>|",
291
+ "|<EXTRA_TOKENS_288>|",
292
+ "|<EXTRA_TOKENS_289>|",
293
+ "|<EXTRA_TOKENS_290>|",
294
+ "|<EXTRA_TOKENS_291>|",
295
+ "|<EXTRA_TOKENS_292>|",
296
+ "|<EXTRA_TOKENS_293>|",
297
+ "|<EXTRA_TOKENS_294>|",
298
+ "|<EXTRA_TOKENS_295>|",
299
+ "|<EXTRA_TOKENS_296>|",
300
+ "|<EXTRA_TOKENS_297>|",
301
+ "|<EXTRA_TOKENS_298>|",
302
+ "|<EXTRA_TOKENS_299>|",
303
+ "|<EXTRA_TOKENS_300>|",
304
+ "|<EXTRA_TOKENS_301>|",
305
+ "|<EXTRA_TOKENS_302>|",
306
+ "|<EXTRA_TOKENS_303>|",
307
+ "|<EXTRA_TOKENS_304>|",
308
+ "|<EXTRA_TOKENS_305>|",
309
+ "|<EXTRA_TOKENS_306>|",
310
+ "|<EXTRA_TOKENS_307>|",
311
+ "|<EXTRA_TOKENS_308>|",
312
+ "|<EXTRA_TOKENS_309>|",
313
+ "|<EXTRA_TOKENS_310>|",
314
+ "|<EXTRA_TOKENS_311>|",
315
+ "|<EXTRA_TOKENS_312>|",
316
+ "|<EXTRA_TOKENS_313>|",
317
+ "|<EXTRA_TOKENS_314>|",
318
+ "|<EXTRA_TOKENS_315>|",
319
+ "|<EXTRA_TOKENS_316>|",
320
+ "|<EXTRA_TOKENS_317>|",
321
+ "|<EXTRA_TOKENS_318>|",
322
+ "|<EXTRA_TOKENS_319>|",
323
+ "|<EXTRA_TOKENS_320>|",
324
+ "|<EXTRA_TOKENS_321>|",
325
+ "|<EXTRA_TOKENS_322>|",
326
+ "|<EXTRA_TOKENS_323>|",
327
+ "|<EXTRA_TOKENS_324>|",
328
+ "|<EXTRA_TOKENS_325>|",
329
+ "|<EXTRA_TOKENS_326>|",
330
+ "|<EXTRA_TOKENS_327>|",
331
+ "|<EXTRA_TOKENS_328>|",
332
+ "|<EXTRA_TOKENS_329>|",
333
+ "|<EXTRA_TOKENS_330>|",
334
+ "|<EXTRA_TOKENS_331>|",
335
+ "|<EXTRA_TOKENS_332>|",
336
+ "|<EXTRA_TOKENS_333>|",
337
+ "|<EXTRA_TOKENS_334>|",
338
+ "|<EXTRA_TOKENS_335>|",
339
+ "|<EXTRA_TOKENS_336>|",
340
+ "|<EXTRA_TOKENS_337>|",
341
+ "|<EXTRA_TOKENS_338>|",
342
+ "|<EXTRA_TOKENS_339>|",
343
+ "|<EXTRA_TOKENS_340>|",
344
+ "|<EXTRA_TOKENS_341>|",
345
+ "|<EXTRA_TOKENS_342>|",
346
+ "|<EXTRA_TOKENS_343>|",
347
+ "|<EXTRA_TOKENS_344>|",
348
+ "|<EXTRA_TOKENS_345>|",
349
+ "|<EXTRA_TOKENS_346>|",
350
+ "|<EXTRA_TOKENS_347>|",
351
+ "|<EXTRA_TOKENS_348>|",
352
+ "|<EXTRA_TOKENS_349>|",
353
+ "|<EXTRA_TOKENS_350>|",
354
+ "|<EXTRA_TOKENS_351>|",
355
+ "|<EXTRA_TOKENS_352>|",
356
+ "|<EXTRA_TOKENS_353>|",
357
+ "|<EXTRA_TOKENS_354>|",
358
+ "|<EXTRA_TOKENS_355>|",
359
+ "|<EXTRA_TOKENS_356>|",
360
+ "|<EXTRA_TOKENS_357>|",
361
+ "|<EXTRA_TOKENS_358>|",
362
+ "|<EXTRA_TOKENS_359>|",
363
+ "|<EXTRA_TOKENS_360>|",
364
+ "|<EXTRA_TOKENS_361>|",
365
+ "|<EXTRA_TOKENS_362>|",
366
+ "|<EXTRA_TOKENS_363>|",
367
+ "|<EXTRA_TOKENS_364>|",
368
+ "|<EXTRA_TOKENS_365>|",
369
+ "|<EXTRA_TOKENS_366>|",
370
+ "|<EXTRA_TOKENS_367>|",
371
+ "|<EXTRA_TOKENS_368>|",
372
+ "|<EXTRA_TOKENS_369>|",
373
+ "|<EXTRA_TOKENS_370>|",
374
+ "|<EXTRA_TOKENS_371>|",
375
+ "|<EXTRA_TOKENS_372>|",
376
+ "|<EXTRA_TOKENS_373>|",
377
+ "|<EXTRA_TOKENS_374>|",
378
+ "|<EXTRA_TOKENS_375>|",
379
+ "|<EXTRA_TOKENS_376>|",
380
+ "|<EXTRA_TOKENS_377>|",
381
+ "|<EXTRA_TOKENS_378>|",
382
+ "|<EXTRA_TOKENS_379>|",
383
+ "|<EXTRA_TOKENS_380>|",
384
+ "|<EXTRA_TOKENS_381>|",
385
+ "|<EXTRA_TOKENS_382>|",
386
+ "|<EXTRA_TOKENS_383>|",
387
+ "|<EXTRA_TOKENS_384>|",
388
+ "|<EXTRA_TOKENS_385>|",
389
+ "|<EXTRA_TOKENS_386>|",
390
+ "|<EXTRA_TOKENS_387>|",
391
+ "|<EXTRA_TOKENS_388>|",
392
+ "|<EXTRA_TOKENS_389>|",
393
+ "|<EXTRA_TOKENS_390>|",
394
+ "|<EXTRA_TOKENS_391>|",
395
+ "|<EXTRA_TOKENS_392>|",
396
+ "|<EXTRA_TOKENS_393>|",
397
+ "|<EXTRA_TOKENS_394>|",
398
+ "|<EXTRA_TOKENS_395>|",
399
+ "|<EXTRA_TOKENS_396>|",
400
+ "|<EXTRA_TOKENS_397>|",
401
+ "|<EXTRA_TOKENS_398>|",
402
+ "|<EXTRA_TOKENS_399>|",
403
+ "|<EXTRA_TOKENS_400>|",
404
+ "|<EXTRA_TOKENS_401>|",
405
+ "|<EXTRA_TOKENS_402>|",
406
+ "|<EXTRA_TOKENS_403>|",
407
+ "|<EXTRA_TOKENS_404>|",
408
+ "|<EXTRA_TOKENS_405>|",
409
+ "|<EXTRA_TOKENS_406>|",
410
+ "|<EXTRA_TOKENS_407>|",
411
+ "|<EXTRA_TOKENS_408>|",
412
+ "|<EXTRA_TOKENS_409>|",
413
+ "|<EXTRA_TOKENS_410>|",
414
+ "|<EXTRA_TOKENS_411>|",
415
+ "|<EXTRA_TOKENS_412>|",
416
+ "|<EXTRA_TOKENS_413>|",
417
+ "|<EXTRA_TOKENS_414>|",
418
+ "|<EXTRA_TOKENS_415>|",
419
+ "|<EXTRA_TOKENS_416>|",
420
+ "|<EXTRA_TOKENS_417>|",
421
+ "<im_start>",
422
+ "<im_end>",
423
+ "<im_patch>",
424
+ "<im_col>",
425
+ "<|image|>"
426
+ ],
427
+ "eos_token": {
428
+ "content": "<|endoftext|>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false
433
+ },
434
+ "pad_token": {
435
+ "content": "<|endoftext|>",
436
+ "lstrip": false,
437
+ "normalized": false,
438
+ "rstrip": false,
439
+ "single_word": false
440
+ }
441
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6248048a83152ce87663c799492fe7e60c8086f3ae51ce7bd255ccc445746fc0
3
+ size 11501432
tokenizer_config.json ADDED
@@ -0,0 +1,3853 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "151646": {
29
+ "content": "|<EXTRA_TOKENS_0>|",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "151647": {
37
+ "content": "|<EXTRA_TOKENS_1>|",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "151648": {
45
+ "content": "|<EXTRA_TOKENS_2>|",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "151649": {
53
+ "content": "|<EXTRA_TOKENS_3>|",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "151650": {
61
+ "content": "|<EXTRA_TOKENS_4>|",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "151651": {
69
+ "content": "|<EXTRA_TOKENS_5>|",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "151652": {
77
+ "content": "|<EXTRA_TOKENS_6>|",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "151653": {
85
+ "content": "|<EXTRA_TOKENS_7>|",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "151654": {
93
+ "content": "|<EXTRA_TOKENS_8>|",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "151655": {
101
+ "content": "|<EXTRA_TOKENS_9>|",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "151656": {
109
+ "content": "|<EXTRA_TOKENS_10>|",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "151657": {
117
+ "content": "|<EXTRA_TOKENS_11>|",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "151658": {
125
+ "content": "|<EXTRA_TOKENS_12>|",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "151659": {
133
+ "content": "|<EXTRA_TOKENS_13>|",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "151660": {
141
+ "content": "|<EXTRA_TOKENS_14>|",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "151661": {
149
+ "content": "|<EXTRA_TOKENS_15>|",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "151662": {
157
+ "content": "|<EXTRA_TOKENS_16>|",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "151663": {
165
+ "content": "|<EXTRA_TOKENS_17>|",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "151664": {
173
+ "content": "|<EXTRA_TOKENS_18>|",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "151665": {
181
+ "content": "|<EXTRA_TOKENS_19>|",
182
+ "lstrip": false,
183
+ "normalized": false,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "151666": {
189
+ "content": "|<EXTRA_TOKENS_20>|",
190
+ "lstrip": false,
191
+ "normalized": false,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "151667": {
197
+ "content": "|<EXTRA_TOKENS_21>|",
198
+ "lstrip": false,
199
+ "normalized": false,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "151668": {
205
+ "content": "|<EXTRA_TOKENS_22>|",
206
+ "lstrip": false,
207
+ "normalized": false,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "151669": {
213
+ "content": "|<EXTRA_TOKENS_23>|",
214
+ "lstrip": false,
215
+ "normalized": false,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "151670": {
221
+ "content": "|<EXTRA_TOKENS_24>|",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "151671": {
229
+ "content": "|<EXTRA_TOKENS_25>|",
230
+ "lstrip": false,
231
+ "normalized": false,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "151672": {
237
+ "content": "|<EXTRA_TOKENS_26>|",
238
+ "lstrip": false,
239
+ "normalized": false,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "151673": {
245
+ "content": "|<EXTRA_TOKENS_27>|",
246
+ "lstrip": false,
247
+ "normalized": false,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "151674": {
253
+ "content": "|<EXTRA_TOKENS_28>|",
254
+ "lstrip": false,
255
+ "normalized": false,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "151675": {
261
+ "content": "|<EXTRA_TOKENS_29>|",
262
+ "lstrip": false,
263
+ "normalized": false,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "151676": {
269
+ "content": "|<EXTRA_TOKENS_30>|",
270
+ "lstrip": false,
271
+ "normalized": false,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "151677": {
277
+ "content": "|<EXTRA_TOKENS_31>|",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "151678": {
285
+ "content": "|<EXTRA_TOKENS_32>|",
286
+ "lstrip": false,
287
+ "normalized": false,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "151679": {
293
+ "content": "|<EXTRA_TOKENS_33>|",
294
+ "lstrip": false,
295
+ "normalized": false,
296
+ "rstrip": false,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "151680": {
301
+ "content": "|<EXTRA_TOKENS_34>|",
302
+ "lstrip": false,
303
+ "normalized": false,
304
+ "rstrip": false,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "151681": {
309
+ "content": "|<EXTRA_TOKENS_35>|",
310
+ "lstrip": false,
311
+ "normalized": false,
312
+ "rstrip": false,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "151682": {
317
+ "content": "|<EXTRA_TOKENS_36>|",
318
+ "lstrip": false,
319
+ "normalized": false,
320
+ "rstrip": false,
321
+ "single_word": false,
322
+ "special": true
323
+ },
324
+ "151683": {
325
+ "content": "|<EXTRA_TOKENS_37>|",
326
+ "lstrip": false,
327
+ "normalized": false,
328
+ "rstrip": false,
329
+ "single_word": false,
330
+ "special": true
331
+ },
332
+ "151684": {
333
+ "content": "|<EXTRA_TOKENS_38>|",
334
+ "lstrip": false,
335
+ "normalized": false,
336
+ "rstrip": false,
337
+ "single_word": false,
338
+ "special": true
339
+ },
340
+ "151685": {
341
+ "content": "|<EXTRA_TOKENS_39>|",
342
+ "lstrip": false,
343
+ "normalized": false,
344
+ "rstrip": false,
345
+ "single_word": false,
346
+ "special": true
347
+ },
348
+ "151686": {
349
+ "content": "|<EXTRA_TOKENS_40>|",
350
+ "lstrip": false,
351
+ "normalized": false,
352
+ "rstrip": false,
353
+ "single_word": false,
354
+ "special": true
355
+ },
356
+ "151687": {
357
+ "content": "|<EXTRA_TOKENS_41>|",
358
+ "lstrip": false,
359
+ "normalized": false,
360
+ "rstrip": false,
361
+ "single_word": false,
362
+ "special": true
363
+ },
364
+ "151688": {
365
+ "content": "|<EXTRA_TOKENS_42>|",
366
+ "lstrip": false,
367
+ "normalized": false,
368
+ "rstrip": false,
369
+ "single_word": false,
370
+ "special": true
371
+ },
372
+ "151689": {
373
+ "content": "|<EXTRA_TOKENS_43>|",
374
+ "lstrip": false,
375
+ "normalized": false,
376
+ "rstrip": false,
377
+ "single_word": false,
378
+ "special": true
379
+ },
380
+ "151690": {
381
+ "content": "|<EXTRA_TOKENS_44>|",
382
+ "lstrip": false,
383
+ "normalized": false,
384
+ "rstrip": false,
385
+ "single_word": false,
386
+ "special": true
387
+ },
388
+ "151691": {
389
+ "content": "|<EXTRA_TOKENS_45>|",
390
+ "lstrip": false,
391
+ "normalized": false,
392
+ "rstrip": false,
393
+ "single_word": false,
394
+ "special": true
395
+ },
396
+ "151692": {
397
+ "content": "|<EXTRA_TOKENS_46>|",
398
+ "lstrip": false,
399
+ "normalized": false,
400
+ "rstrip": false,
401
+ "single_word": false,
402
+ "special": true
403
+ },
404
+ "151693": {
405
+ "content": "|<EXTRA_TOKENS_47>|",
406
+ "lstrip": false,
407
+ "normalized": false,
408
+ "rstrip": false,
409
+ "single_word": false,
410
+ "special": true
411
+ },
412
+ "151694": {
413
+ "content": "|<EXTRA_TOKENS_48>|",
414
+ "lstrip": false,
415
+ "normalized": false,
416
+ "rstrip": false,
417
+ "single_word": false,
418
+ "special": true
419
+ },
420
+ "151695": {
421
+ "content": "|<EXTRA_TOKENS_49>|",
422
+ "lstrip": false,
423
+ "normalized": false,
424
+ "rstrip": false,
425
+ "single_word": false,
426
+ "special": true
427
+ },
428
+ "151696": {
429
+ "content": "|<EXTRA_TOKENS_50>|",
430
+ "lstrip": false,
431
+ "normalized": false,
432
+ "rstrip": false,
433
+ "single_word": false,
434
+ "special": true
435
+ },
436
+ "151697": {
437
+ "content": "|<EXTRA_TOKENS_51>|",
438
+ "lstrip": false,
439
+ "normalized": false,
440
+ "rstrip": false,
441
+ "single_word": false,
442
+ "special": true
443
+ },
444
+ "151698": {
445
+ "content": "|<EXTRA_TOKENS_52>|",
446
+ "lstrip": false,
447
+ "normalized": false,
448
+ "rstrip": false,
449
+ "single_word": false,
450
+ "special": true
451
+ },
452
+ "151699": {
453
+ "content": "|<EXTRA_TOKENS_53>|",
454
+ "lstrip": false,
455
+ "normalized": false,
456
+ "rstrip": false,
457
+ "single_word": false,
458
+ "special": true
459
+ },
460
+ "151700": {
461
+ "content": "|<EXTRA_TOKENS_54>|",
462
+ "lstrip": false,
463
+ "normalized": false,
464
+ "rstrip": false,
465
+ "single_word": false,
466
+ "special": true
467
+ },
468
+ "151701": {
469
+ "content": "|<EXTRA_TOKENS_55>|",
470
+ "lstrip": false,
471
+ "normalized": false,
472
+ "rstrip": false,
473
+ "single_word": false,
474
+ "special": true
475
+ },
476
+ "151702": {
477
+ "content": "|<EXTRA_TOKENS_56>|",
478
+ "lstrip": false,
479
+ "normalized": false,
480
+ "rstrip": false,
481
+ "single_word": false,
482
+ "special": true
483
+ },
484
+ "151703": {
485
+ "content": "|<EXTRA_TOKENS_57>|",
486
+ "lstrip": false,
487
+ "normalized": false,
488
+ "rstrip": false,
489
+ "single_word": false,
490
+ "special": true
491
+ },
492
+ "151704": {
493
+ "content": "|<EXTRA_TOKENS_58>|",
494
+ "lstrip": false,
495
+ "normalized": false,
496
+ "rstrip": false,
497
+ "single_word": false,
498
+ "special": true
499
+ },
500
+ "151705": {
501
+ "content": "|<EXTRA_TOKENS_59>|",
502
+ "lstrip": false,
503
+ "normalized": false,
504
+ "rstrip": false,
505
+ "single_word": false,
506
+ "special": true
507
+ },
508
+ "151706": {
509
+ "content": "|<EXTRA_TOKENS_60>|",
510
+ "lstrip": false,
511
+ "normalized": false,
512
+ "rstrip": false,
513
+ "single_word": false,
514
+ "special": true
515
+ },
516
+ "151707": {
517
+ "content": "|<EXTRA_TOKENS_61>|",
518
+ "lstrip": false,
519
+ "normalized": false,
520
+ "rstrip": false,
521
+ "single_word": false,
522
+ "special": true
523
+ },
524
+ "151708": {
525
+ "content": "|<EXTRA_TOKENS_62>|",
526
+ "lstrip": false,
527
+ "normalized": false,
528
+ "rstrip": false,
529
+ "single_word": false,
530
+ "special": true
531
+ },
532
+ "151709": {
533
+ "content": "|<EXTRA_TOKENS_63>|",
534
+ "lstrip": false,
535
+ "normalized": false,
536
+ "rstrip": false,
537
+ "single_word": false,
538
+ "special": true
539
+ },
540
+ "151710": {
541
+ "content": "|<EXTRA_TOKENS_64>|",
542
+ "lstrip": false,
543
+ "normalized": false,
544
+ "rstrip": false,
545
+ "single_word": false,
546
+ "special": true
547
+ },
548
+ "151711": {
549
+ "content": "|<EXTRA_TOKENS_65>|",
550
+ "lstrip": false,
551
+ "normalized": false,
552
+ "rstrip": false,
553
+ "single_word": false,
554
+ "special": true
555
+ },
556
+ "151712": {
557
+ "content": "|<EXTRA_TOKENS_66>|",
558
+ "lstrip": false,
559
+ "normalized": false,
560
+ "rstrip": false,
561
+ "single_word": false,
562
+ "special": true
563
+ },
564
+ "151713": {
565
+ "content": "|<EXTRA_TOKENS_67>|",
566
+ "lstrip": false,
567
+ "normalized": false,
568
+ "rstrip": false,
569
+ "single_word": false,
570
+ "special": true
571
+ },
572
+ "151714": {
573
+ "content": "|<EXTRA_TOKENS_68>|",
574
+ "lstrip": false,
575
+ "normalized": false,
576
+ "rstrip": false,
577
+ "single_word": false,
578
+ "special": true
579
+ },
580
+ "151715": {
581
+ "content": "|<EXTRA_TOKENS_69>|",
582
+ "lstrip": false,
583
+ "normalized": false,
584
+ "rstrip": false,
585
+ "single_word": false,
586
+ "special": true
587
+ },
588
+ "151716": {
589
+ "content": "|<EXTRA_TOKENS_70>|",
590
+ "lstrip": false,
591
+ "normalized": false,
592
+ "rstrip": false,
593
+ "single_word": false,
594
+ "special": true
595
+ },
596
+ "151717": {
597
+ "content": "|<EXTRA_TOKENS_71>|",
598
+ "lstrip": false,
599
+ "normalized": false,
600
+ "rstrip": false,
601
+ "single_word": false,
602
+ "special": true
603
+ },
604
+ "151718": {
605
+ "content": "|<EXTRA_TOKENS_72>|",
606
+ "lstrip": false,
607
+ "normalized": false,
608
+ "rstrip": false,
609
+ "single_word": false,
610
+ "special": true
611
+ },
612
+ "151719": {
613
+ "content": "|<EXTRA_TOKENS_73>|",
614
+ "lstrip": false,
615
+ "normalized": false,
616
+ "rstrip": false,
617
+ "single_word": false,
618
+ "special": true
619
+ },
620
+ "151720": {
621
+ "content": "|<EXTRA_TOKENS_74>|",
622
+ "lstrip": false,
623
+ "normalized": false,
624
+ "rstrip": false,
625
+ "single_word": false,
626
+ "special": true
627
+ },
628
+ "151721": {
629
+ "content": "|<EXTRA_TOKENS_75>|",
630
+ "lstrip": false,
631
+ "normalized": false,
632
+ "rstrip": false,
633
+ "single_word": false,
634
+ "special": true
635
+ },
636
+ "151722": {
637
+ "content": "|<EXTRA_TOKENS_76>|",
638
+ "lstrip": false,
639
+ "normalized": false,
640
+ "rstrip": false,
641
+ "single_word": false,
642
+ "special": true
643
+ },
644
+ "151723": {
645
+ "content": "|<EXTRA_TOKENS_77>|",
646
+ "lstrip": false,
647
+ "normalized": false,
648
+ "rstrip": false,
649
+ "single_word": false,
650
+ "special": true
651
+ },
652
+ "151724": {
653
+ "content": "|<EXTRA_TOKENS_78>|",
654
+ "lstrip": false,
655
+ "normalized": false,
656
+ "rstrip": false,
657
+ "single_word": false,
658
+ "special": true
659
+ },
660
+ "151725": {
661
+ "content": "|<EXTRA_TOKENS_79>|",
662
+ "lstrip": false,
663
+ "normalized": false,
664
+ "rstrip": false,
665
+ "single_word": false,
666
+ "special": true
667
+ },
668
+ "151726": {
669
+ "content": "|<EXTRA_TOKENS_80>|",
670
+ "lstrip": false,
671
+ "normalized": false,
672
+ "rstrip": false,
673
+ "single_word": false,
674
+ "special": true
675
+ },
676
+ "151727": {
677
+ "content": "|<EXTRA_TOKENS_81>|",
678
+ "lstrip": false,
679
+ "normalized": false,
680
+ "rstrip": false,
681
+ "single_word": false,
682
+ "special": true
683
+ },
684
+ "151728": {
685
+ "content": "|<EXTRA_TOKENS_82>|",
686
+ "lstrip": false,
687
+ "normalized": false,
688
+ "rstrip": false,
689
+ "single_word": false,
690
+ "special": true
691
+ },
692
+ "151729": {
693
+ "content": "|<EXTRA_TOKENS_83>|",
694
+ "lstrip": false,
695
+ "normalized": false,
696
+ "rstrip": false,
697
+ "single_word": false,
698
+ "special": true
699
+ },
700
+ "151730": {
701
+ "content": "|<EXTRA_TOKENS_84>|",
702
+ "lstrip": false,
703
+ "normalized": false,
704
+ "rstrip": false,
705
+ "single_word": false,
706
+ "special": true
707
+ },
708
+ "151731": {
709
+ "content": "|<EXTRA_TOKENS_85>|",
710
+ "lstrip": false,
711
+ "normalized": false,
712
+ "rstrip": false,
713
+ "single_word": false,
714
+ "special": true
715
+ },
716
+ "151732": {
717
+ "content": "|<EXTRA_TOKENS_86>|",
718
+ "lstrip": false,
719
+ "normalized": false,
720
+ "rstrip": false,
721
+ "single_word": false,
722
+ "special": true
723
+ },
724
+ "151733": {
725
+ "content": "|<EXTRA_TOKENS_87>|",
726
+ "lstrip": false,
727
+ "normalized": false,
728
+ "rstrip": false,
729
+ "single_word": false,
730
+ "special": true
731
+ },
732
+ "151734": {
733
+ "content": "|<EXTRA_TOKENS_88>|",
734
+ "lstrip": false,
735
+ "normalized": false,
736
+ "rstrip": false,
737
+ "single_word": false,
738
+ "special": true
739
+ },
740
+ "151735": {
741
+ "content": "|<EXTRA_TOKENS_89>|",
742
+ "lstrip": false,
743
+ "normalized": false,
744
+ "rstrip": false,
745
+ "single_word": false,
746
+ "special": true
747
+ },
748
+ "151736": {
749
+ "content": "|<EXTRA_TOKENS_90>|",
750
+ "lstrip": false,
751
+ "normalized": false,
752
+ "rstrip": false,
753
+ "single_word": false,
754
+ "special": true
755
+ },
756
+ "151737": {
757
+ "content": "|<EXTRA_TOKENS_91>|",
758
+ "lstrip": false,
759
+ "normalized": false,
760
+ "rstrip": false,
761
+ "single_word": false,
762
+ "special": true
763
+ },
764
+ "151738": {
765
+ "content": "|<EXTRA_TOKENS_92>|",
766
+ "lstrip": false,
767
+ "normalized": false,
768
+ "rstrip": false,
769
+ "single_word": false,
770
+ "special": true
771
+ },
772
+ "151739": {
773
+ "content": "|<EXTRA_TOKENS_93>|",
774
+ "lstrip": false,
775
+ "normalized": false,
776
+ "rstrip": false,
777
+ "single_word": false,
778
+ "special": true
779
+ },
780
+ "151740": {
781
+ "content": "|<EXTRA_TOKENS_94>|",
782
+ "lstrip": false,
783
+ "normalized": false,
784
+ "rstrip": false,
785
+ "single_word": false,
786
+ "special": true
787
+ },
788
+ "151741": {
789
+ "content": "|<EXTRA_TOKENS_95>|",
790
+ "lstrip": false,
791
+ "normalized": false,
792
+ "rstrip": false,
793
+ "single_word": false,
794
+ "special": true
795
+ },
796
+ "151742": {
797
+ "content": "|<EXTRA_TOKENS_96>|",
798
+ "lstrip": false,
799
+ "normalized": false,
800
+ "rstrip": false,
801
+ "single_word": false,
802
+ "special": true
803
+ },
804
+ "151743": {
805
+ "content": "|<EXTRA_TOKENS_97>|",
806
+ "lstrip": false,
807
+ "normalized": false,
808
+ "rstrip": false,
809
+ "single_word": false,
810
+ "special": true
811
+ },
812
+ "151744": {
813
+ "content": "|<EXTRA_TOKENS_98>|",
814
+ "lstrip": false,
815
+ "normalized": false,
816
+ "rstrip": false,
817
+ "single_word": false,
818
+ "special": true
819
+ },
820
+ "151745": {
821
+ "content": "|<EXTRA_TOKENS_99>|",
822
+ "lstrip": false,
823
+ "normalized": false,
824
+ "rstrip": false,
825
+ "single_word": false,
826
+ "special": true
827
+ },
828
+ "151746": {
829
+ "content": "|<EXTRA_TOKENS_100>|",
830
+ "lstrip": false,
831
+ "normalized": false,
832
+ "rstrip": false,
833
+ "single_word": false,
834
+ "special": true
835
+ },
836
+ "151747": {
837
+ "content": "|<EXTRA_TOKENS_101>|",
838
+ "lstrip": false,
839
+ "normalized": false,
840
+ "rstrip": false,
841
+ "single_word": false,
842
+ "special": true
843
+ },
844
+ "151748": {
845
+ "content": "|<EXTRA_TOKENS_102>|",
846
+ "lstrip": false,
847
+ "normalized": false,
848
+ "rstrip": false,
849
+ "single_word": false,
850
+ "special": true
851
+ },
852
+ "151749": {
853
+ "content": "|<EXTRA_TOKENS_103>|",
854
+ "lstrip": false,
855
+ "normalized": false,
856
+ "rstrip": false,
857
+ "single_word": false,
858
+ "special": true
859
+ },
860
+ "151750": {
861
+ "content": "|<EXTRA_TOKENS_104>|",
862
+ "lstrip": false,
863
+ "normalized": false,
864
+ "rstrip": false,
865
+ "single_word": false,
866
+ "special": true
867
+ },
868
+ "151751": {
869
+ "content": "|<EXTRA_TOKENS_105>|",
870
+ "lstrip": false,
871
+ "normalized": false,
872
+ "rstrip": false,
873
+ "single_word": false,
874
+ "special": true
875
+ },
876
+ "151752": {
877
+ "content": "|<EXTRA_TOKENS_106>|",
878
+ "lstrip": false,
879
+ "normalized": false,
880
+ "rstrip": false,
881
+ "single_word": false,
882
+ "special": true
883
+ },
884
+ "151753": {
885
+ "content": "|<EXTRA_TOKENS_107>|",
886
+ "lstrip": false,
887
+ "normalized": false,
888
+ "rstrip": false,
889
+ "single_word": false,
890
+ "special": true
891
+ },
892
+ "151754": {
893
+ "content": "|<EXTRA_TOKENS_108>|",
894
+ "lstrip": false,
895
+ "normalized": false,
896
+ "rstrip": false,
897
+ "single_word": false,
898
+ "special": true
899
+ },
900
+ "151755": {
901
+ "content": "|<EXTRA_TOKENS_109>|",
902
+ "lstrip": false,
903
+ "normalized": false,
904
+ "rstrip": false,
905
+ "single_word": false,
906
+ "special": true
907
+ },
908
+ "151756": {
909
+ "content": "|<EXTRA_TOKENS_110>|",
910
+ "lstrip": false,
911
+ "normalized": false,
912
+ "rstrip": false,
913
+ "single_word": false,
914
+ "special": true
915
+ },
916
+ "151757": {
917
+ "content": "|<EXTRA_TOKENS_111>|",
918
+ "lstrip": false,
919
+ "normalized": false,
920
+ "rstrip": false,
921
+ "single_word": false,
922
+ "special": true
923
+ },
924
+ "151758": {
925
+ "content": "|<EXTRA_TOKENS_112>|",
926
+ "lstrip": false,
927
+ "normalized": false,
928
+ "rstrip": false,
929
+ "single_word": false,
930
+ "special": true
931
+ },
932
+ "151759": {
933
+ "content": "|<EXTRA_TOKENS_113>|",
934
+ "lstrip": false,
935
+ "normalized": false,
936
+ "rstrip": false,
937
+ "single_word": false,
938
+ "special": true
939
+ },
940
+ "151760": {
941
+ "content": "|<EXTRA_TOKENS_114>|",
942
+ "lstrip": false,
943
+ "normalized": false,
944
+ "rstrip": false,
945
+ "single_word": false,
946
+ "special": true
947
+ },
948
+ "151761": {
949
+ "content": "|<EXTRA_TOKENS_115>|",
950
+ "lstrip": false,
951
+ "normalized": false,
952
+ "rstrip": false,
953
+ "single_word": false,
954
+ "special": true
955
+ },
956
+ "151762": {
957
+ "content": "|<EXTRA_TOKENS_116>|",
958
+ "lstrip": false,
959
+ "normalized": false,
960
+ "rstrip": false,
961
+ "single_word": false,
962
+ "special": true
963
+ },
964
+ "151763": {
965
+ "content": "|<EXTRA_TOKENS_117>|",
966
+ "lstrip": false,
967
+ "normalized": false,
968
+ "rstrip": false,
969
+ "single_word": false,
970
+ "special": true
971
+ },
972
+ "151764": {
973
+ "content": "|<EXTRA_TOKENS_118>|",
974
+ "lstrip": false,
975
+ "normalized": false,
976
+ "rstrip": false,
977
+ "single_word": false,
978
+ "special": true
979
+ },
980
+ "151765": {
981
+ "content": "|<EXTRA_TOKENS_119>|",
982
+ "lstrip": false,
983
+ "normalized": false,
984
+ "rstrip": false,
985
+ "single_word": false,
986
+ "special": true
987
+ },
988
+ "151766": {
989
+ "content": "|<EXTRA_TOKENS_120>|",
990
+ "lstrip": false,
991
+ "normalized": false,
992
+ "rstrip": false,
993
+ "single_word": false,
994
+ "special": true
995
+ },
996
+ "151767": {
997
+ "content": "|<EXTRA_TOKENS_121>|",
998
+ "lstrip": false,
999
+ "normalized": false,
1000
+ "rstrip": false,
1001
+ "single_word": false,
1002
+ "special": true
1003
+ },
1004
+ "151768": {
1005
+ "content": "|<EXTRA_TOKENS_122>|",
1006
+ "lstrip": false,
1007
+ "normalized": false,
1008
+ "rstrip": false,
1009
+ "single_word": false,
1010
+ "special": true
1011
+ },
1012
+ "151769": {
1013
+ "content": "|<EXTRA_TOKENS_123>|",
1014
+ "lstrip": false,
1015
+ "normalized": false,
1016
+ "rstrip": false,
1017
+ "single_word": false,
1018
+ "special": true
1019
+ },
1020
+ "151770": {
1021
+ "content": "|<EXTRA_TOKENS_124>|",
1022
+ "lstrip": false,
1023
+ "normalized": false,
1024
+ "rstrip": false,
1025
+ "single_word": false,
1026
+ "special": true
1027
+ },
1028
+ "151771": {
1029
+ "content": "|<EXTRA_TOKENS_125>|",
1030
+ "lstrip": false,
1031
+ "normalized": false,
1032
+ "rstrip": false,
1033
+ "single_word": false,
1034
+ "special": true
1035
+ },
1036
+ "151772": {
1037
+ "content": "|<EXTRA_TOKENS_126>|",
1038
+ "lstrip": false,
1039
+ "normalized": false,
1040
+ "rstrip": false,
1041
+ "single_word": false,
1042
+ "special": true
1043
+ },
1044
+ "151773": {
1045
+ "content": "|<EXTRA_TOKENS_127>|",
1046
+ "lstrip": false,
1047
+ "normalized": false,
1048
+ "rstrip": false,
1049
+ "single_word": false,
1050
+ "special": true
1051
+ },
1052
+ "151774": {
1053
+ "content": "|<EXTRA_TOKENS_128>|",
1054
+ "lstrip": false,
1055
+ "normalized": false,
1056
+ "rstrip": false,
1057
+ "single_word": false,
1058
+ "special": true
1059
+ },
1060
+ "151775": {
1061
+ "content": "|<EXTRA_TOKENS_129>|",
1062
+ "lstrip": false,
1063
+ "normalized": false,
1064
+ "rstrip": false,
1065
+ "single_word": false,
1066
+ "special": true
1067
+ },
1068
+ "151776": {
1069
+ "content": "|<EXTRA_TOKENS_130>|",
1070
+ "lstrip": false,
1071
+ "normalized": false,
1072
+ "rstrip": false,
1073
+ "single_word": false,
1074
+ "special": true
1075
+ },
1076
+ "151777": {
1077
+ "content": "|<EXTRA_TOKENS_131>|",
1078
+ "lstrip": false,
1079
+ "normalized": false,
1080
+ "rstrip": false,
1081
+ "single_word": false,
1082
+ "special": true
1083
+ },
1084
+ "151778": {
1085
+ "content": "|<EXTRA_TOKENS_132>|",
1086
+ "lstrip": false,
1087
+ "normalized": false,
1088
+ "rstrip": false,
1089
+ "single_word": false,
1090
+ "special": true
1091
+ },
1092
+ "151779": {
1093
+ "content": "|<EXTRA_TOKENS_133>|",
1094
+ "lstrip": false,
1095
+ "normalized": false,
1096
+ "rstrip": false,
1097
+ "single_word": false,
1098
+ "special": true
1099
+ },
1100
+ "151780": {
1101
+ "content": "|<EXTRA_TOKENS_134>|",
1102
+ "lstrip": false,
1103
+ "normalized": false,
1104
+ "rstrip": false,
1105
+ "single_word": false,
1106
+ "special": true
1107
+ },
1108
+ "151781": {
1109
+ "content": "|<EXTRA_TOKENS_135>|",
1110
+ "lstrip": false,
1111
+ "normalized": false,
1112
+ "rstrip": false,
1113
+ "single_word": false,
1114
+ "special": true
1115
+ },
1116
+ "151782": {
1117
+ "content": "|<EXTRA_TOKENS_136>|",
1118
+ "lstrip": false,
1119
+ "normalized": false,
1120
+ "rstrip": false,
1121
+ "single_word": false,
1122
+ "special": true
1123
+ },
1124
+ "151783": {
1125
+ "content": "|<EXTRA_TOKENS_137>|",
1126
+ "lstrip": false,
1127
+ "normalized": false,
1128
+ "rstrip": false,
1129
+ "single_word": false,
1130
+ "special": true
1131
+ },
1132
+ "151784": {
1133
+ "content": "|<EXTRA_TOKENS_138>|",
1134
+ "lstrip": false,
1135
+ "normalized": false,
1136
+ "rstrip": false,
1137
+ "single_word": false,
1138
+ "special": true
1139
+ },
1140
+ "151785": {
1141
+ "content": "|<EXTRA_TOKENS_139>|",
1142
+ "lstrip": false,
1143
+ "normalized": false,
1144
+ "rstrip": false,
1145
+ "single_word": false,
1146
+ "special": true
1147
+ },
1148
+ "151786": {
1149
+ "content": "|<EXTRA_TOKENS_140>|",
1150
+ "lstrip": false,
1151
+ "normalized": false,
1152
+ "rstrip": false,
1153
+ "single_word": false,
1154
+ "special": true
1155
+ },
1156
+ "151787": {
1157
+ "content": "|<EXTRA_TOKENS_141>|",
1158
+ "lstrip": false,
1159
+ "normalized": false,
1160
+ "rstrip": false,
1161
+ "single_word": false,
1162
+ "special": true
1163
+ },
1164
+ "151788": {
1165
+ "content": "|<EXTRA_TOKENS_142>|",
1166
+ "lstrip": false,
1167
+ "normalized": false,
1168
+ "rstrip": false,
1169
+ "single_word": false,
1170
+ "special": true
1171
+ },
1172
+ "151789": {
1173
+ "content": "|<EXTRA_TOKENS_143>|",
1174
+ "lstrip": false,
1175
+ "normalized": false,
1176
+ "rstrip": false,
1177
+ "single_word": false,
1178
+ "special": true
1179
+ },
1180
+ "151790": {
1181
+ "content": "|<EXTRA_TOKENS_144>|",
1182
+ "lstrip": false,
1183
+ "normalized": false,
1184
+ "rstrip": false,
1185
+ "single_word": false,
1186
+ "special": true
1187
+ },
1188
+ "151791": {
1189
+ "content": "|<EXTRA_TOKENS_145>|",
1190
+ "lstrip": false,
1191
+ "normalized": false,
1192
+ "rstrip": false,
1193
+ "single_word": false,
1194
+ "special": true
1195
+ },
1196
+ "151792": {
1197
+ "content": "|<EXTRA_TOKENS_146>|",
1198
+ "lstrip": false,
1199
+ "normalized": false,
1200
+ "rstrip": false,
1201
+ "single_word": false,
1202
+ "special": true
1203
+ },
1204
+ "151793": {
1205
+ "content": "|<EXTRA_TOKENS_147>|",
1206
+ "lstrip": false,
1207
+ "normalized": false,
1208
+ "rstrip": false,
1209
+ "single_word": false,
1210
+ "special": true
1211
+ },
1212
+ "151794": {
1213
+ "content": "|<EXTRA_TOKENS_148>|",
1214
+ "lstrip": false,
1215
+ "normalized": false,
1216
+ "rstrip": false,
1217
+ "single_word": false,
1218
+ "special": true
1219
+ },
1220
+ "151795": {
1221
+ "content": "|<EXTRA_TOKENS_149>|",
1222
+ "lstrip": false,
1223
+ "normalized": false,
1224
+ "rstrip": false,
1225
+ "single_word": false,
1226
+ "special": true
1227
+ },
1228
+ "151796": {
1229
+ "content": "|<EXTRA_TOKENS_150>|",
1230
+ "lstrip": false,
1231
+ "normalized": false,
1232
+ "rstrip": false,
1233
+ "single_word": false,
1234
+ "special": true
1235
+ },
1236
+ "151797": {
1237
+ "content": "|<EXTRA_TOKENS_151>|",
1238
+ "lstrip": false,
1239
+ "normalized": false,
1240
+ "rstrip": false,
1241
+ "single_word": false,
1242
+ "special": true
1243
+ },
1244
+ "151798": {
1245
+ "content": "|<EXTRA_TOKENS_152>|",
1246
+ "lstrip": false,
1247
+ "normalized": false,
1248
+ "rstrip": false,
1249
+ "single_word": false,
1250
+ "special": true
1251
+ },
1252
+ "151799": {
1253
+ "content": "|<EXTRA_TOKENS_153>|",
1254
+ "lstrip": false,
1255
+ "normalized": false,
1256
+ "rstrip": false,
1257
+ "single_word": false,
1258
+ "special": true
1259
+ },
1260
+ "151800": {
1261
+ "content": "|<EXTRA_TOKENS_154>|",
1262
+ "lstrip": false,
1263
+ "normalized": false,
1264
+ "rstrip": false,
1265
+ "single_word": false,
1266
+ "special": true
1267
+ },
1268
+ "151801": {
1269
+ "content": "|<EXTRA_TOKENS_155>|",
1270
+ "lstrip": false,
1271
+ "normalized": false,
1272
+ "rstrip": false,
1273
+ "single_word": false,
1274
+ "special": true
1275
+ },
1276
+ "151802": {
1277
+ "content": "|<EXTRA_TOKENS_156>|",
1278
+ "lstrip": false,
1279
+ "normalized": false,
1280
+ "rstrip": false,
1281
+ "single_word": false,
1282
+ "special": true
1283
+ },
1284
+ "151803": {
1285
+ "content": "|<EXTRA_TOKENS_157>|",
1286
+ "lstrip": false,
1287
+ "normalized": false,
1288
+ "rstrip": false,
1289
+ "single_word": false,
1290
+ "special": true
1291
+ },
1292
+ "151804": {
1293
+ "content": "|<EXTRA_TOKENS_158>|",
1294
+ "lstrip": false,
1295
+ "normalized": false,
1296
+ "rstrip": false,
1297
+ "single_word": false,
1298
+ "special": true
1299
+ },
1300
+ "151805": {
1301
+ "content": "|<EXTRA_TOKENS_159>|",
1302
+ "lstrip": false,
1303
+ "normalized": false,
1304
+ "rstrip": false,
1305
+ "single_word": false,
1306
+ "special": true
1307
+ },
1308
+ "151806": {
1309
+ "content": "|<EXTRA_TOKENS_160>|",
1310
+ "lstrip": false,
1311
+ "normalized": false,
1312
+ "rstrip": false,
1313
+ "single_word": false,
1314
+ "special": true
1315
+ },
1316
+ "151807": {
1317
+ "content": "|<EXTRA_TOKENS_161>|",
1318
+ "lstrip": false,
1319
+ "normalized": false,
1320
+ "rstrip": false,
1321
+ "single_word": false,
1322
+ "special": true
1323
+ },
1324
+ "151808": {
1325
+ "content": "|<EXTRA_TOKENS_162>|",
1326
+ "lstrip": false,
1327
+ "normalized": false,
1328
+ "rstrip": false,
1329
+ "single_word": false,
1330
+ "special": true
1331
+ },
1332
+ "151809": {
1333
+ "content": "|<EXTRA_TOKENS_163>|",
1334
+ "lstrip": false,
1335
+ "normalized": false,
1336
+ "rstrip": false,
1337
+ "single_word": false,
1338
+ "special": true
1339
+ },
1340
+ "151810": {
1341
+ "content": "|<EXTRA_TOKENS_164>|",
1342
+ "lstrip": false,
1343
+ "normalized": false,
1344
+ "rstrip": false,
1345
+ "single_word": false,
1346
+ "special": true
1347
+ },
1348
+ "151811": {
1349
+ "content": "|<EXTRA_TOKENS_165>|",
1350
+ "lstrip": false,
1351
+ "normalized": false,
1352
+ "rstrip": false,
1353
+ "single_word": false,
1354
+ "special": true
1355
+ },
1356
+ "151812": {
1357
+ "content": "|<EXTRA_TOKENS_166>|",
1358
+ "lstrip": false,
1359
+ "normalized": false,
1360
+ "rstrip": false,
1361
+ "single_word": false,
1362
+ "special": true
1363
+ },
1364
+ "151813": {
1365
+ "content": "|<EXTRA_TOKENS_167>|",
1366
+ "lstrip": false,
1367
+ "normalized": false,
1368
+ "rstrip": false,
1369
+ "single_word": false,
1370
+ "special": true
1371
+ },
1372
+ "151814": {
1373
+ "content": "|<EXTRA_TOKENS_168>|",
1374
+ "lstrip": false,
1375
+ "normalized": false,
1376
+ "rstrip": false,
1377
+ "single_word": false,
1378
+ "special": true
1379
+ },
1380
+ "151815": {
1381
+ "content": "|<EXTRA_TOKENS_169>|",
1382
+ "lstrip": false,
1383
+ "normalized": false,
1384
+ "rstrip": false,
1385
+ "single_word": false,
1386
+ "special": true
1387
+ },
1388
+ "151816": {
1389
+ "content": "|<EXTRA_TOKENS_170>|",
1390
+ "lstrip": false,
1391
+ "normalized": false,
1392
+ "rstrip": false,
1393
+ "single_word": false,
1394
+ "special": true
1395
+ },
1396
+ "151817": {
1397
+ "content": "|<EXTRA_TOKENS_171>|",
1398
+ "lstrip": false,
1399
+ "normalized": false,
1400
+ "rstrip": false,
1401
+ "single_word": false,
1402
+ "special": true
1403
+ },
1404
+ "151818": {
1405
+ "content": "|<EXTRA_TOKENS_172>|",
1406
+ "lstrip": false,
1407
+ "normalized": false,
1408
+ "rstrip": false,
1409
+ "single_word": false,
1410
+ "special": true
1411
+ },
1412
+ "151819": {
1413
+ "content": "|<EXTRA_TOKENS_173>|",
1414
+ "lstrip": false,
1415
+ "normalized": false,
1416
+ "rstrip": false,
1417
+ "single_word": false,
1418
+ "special": true
1419
+ },
1420
+ "151820": {
1421
+ "content": "|<EXTRA_TOKENS_174>|",
1422
+ "lstrip": false,
1423
+ "normalized": false,
1424
+ "rstrip": false,
1425
+ "single_word": false,
1426
+ "special": true
1427
+ },
1428
+ "151821": {
1429
+ "content": "|<EXTRA_TOKENS_175>|",
1430
+ "lstrip": false,
1431
+ "normalized": false,
1432
+ "rstrip": false,
1433
+ "single_word": false,
1434
+ "special": true
1435
+ },
1436
+ "151822": {
1437
+ "content": "|<EXTRA_TOKENS_176>|",
1438
+ "lstrip": false,
1439
+ "normalized": false,
1440
+ "rstrip": false,
1441
+ "single_word": false,
1442
+ "special": true
1443
+ },
1444
+ "151823": {
1445
+ "content": "|<EXTRA_TOKENS_177>|",
1446
+ "lstrip": false,
1447
+ "normalized": false,
1448
+ "rstrip": false,
1449
+ "single_word": false,
1450
+ "special": true
1451
+ },
1452
+ "151824": {
1453
+ "content": "|<EXTRA_TOKENS_178>|",
1454
+ "lstrip": false,
1455
+ "normalized": false,
1456
+ "rstrip": false,
1457
+ "single_word": false,
1458
+ "special": true
1459
+ },
1460
+ "151825": {
1461
+ "content": "|<EXTRA_TOKENS_179>|",
1462
+ "lstrip": false,
1463
+ "normalized": false,
1464
+ "rstrip": false,
1465
+ "single_word": false,
1466
+ "special": true
1467
+ },
1468
+ "151826": {
1469
+ "content": "|<EXTRA_TOKENS_180>|",
1470
+ "lstrip": false,
1471
+ "normalized": false,
1472
+ "rstrip": false,
1473
+ "single_word": false,
1474
+ "special": true
1475
+ },
1476
+ "151827": {
1477
+ "content": "|<EXTRA_TOKENS_181>|",
1478
+ "lstrip": false,
1479
+ "normalized": false,
1480
+ "rstrip": false,
1481
+ "single_word": false,
1482
+ "special": true
1483
+ },
1484
+ "151828": {
1485
+ "content": "|<EXTRA_TOKENS_182>|",
1486
+ "lstrip": false,
1487
+ "normalized": false,
1488
+ "rstrip": false,
1489
+ "single_word": false,
1490
+ "special": true
1491
+ },
1492
+ "151829": {
1493
+ "content": "|<EXTRA_TOKENS_183>|",
1494
+ "lstrip": false,
1495
+ "normalized": false,
1496
+ "rstrip": false,
1497
+ "single_word": false,
1498
+ "special": true
1499
+ },
1500
+ "151830": {
1501
+ "content": "|<EXTRA_TOKENS_184>|",
1502
+ "lstrip": false,
1503
+ "normalized": false,
1504
+ "rstrip": false,
1505
+ "single_word": false,
1506
+ "special": true
1507
+ },
1508
+ "151831": {
1509
+ "content": "|<EXTRA_TOKENS_185>|",
1510
+ "lstrip": false,
1511
+ "normalized": false,
1512
+ "rstrip": false,
1513
+ "single_word": false,
1514
+ "special": true
1515
+ },
1516
+ "151832": {
1517
+ "content": "|<EXTRA_TOKENS_186>|",
1518
+ "lstrip": false,
1519
+ "normalized": false,
1520
+ "rstrip": false,
1521
+ "single_word": false,
1522
+ "special": true
1523
+ },
1524
+ "151833": {
1525
+ "content": "|<EXTRA_TOKENS_187>|",
1526
+ "lstrip": false,
1527
+ "normalized": false,
1528
+ "rstrip": false,
1529
+ "single_word": false,
1530
+ "special": true
1531
+ },
1532
+ "151834": {
1533
+ "content": "|<EXTRA_TOKENS_188>|",
1534
+ "lstrip": false,
1535
+ "normalized": false,
1536
+ "rstrip": false,
1537
+ "single_word": false,
1538
+ "special": true
1539
+ },
1540
+ "151835": {
1541
+ "content": "|<EXTRA_TOKENS_189>|",
1542
+ "lstrip": false,
1543
+ "normalized": false,
1544
+ "rstrip": false,
1545
+ "single_word": false,
1546
+ "special": true
1547
+ },
1548
+ "151836": {
1549
+ "content": "|<EXTRA_TOKENS_190>|",
1550
+ "lstrip": false,
1551
+ "normalized": false,
1552
+ "rstrip": false,
1553
+ "single_word": false,
1554
+ "special": true
1555
+ },
1556
+ "151837": {
1557
+ "content": "|<EXTRA_TOKENS_191>|",
1558
+ "lstrip": false,
1559
+ "normalized": false,
1560
+ "rstrip": false,
1561
+ "single_word": false,
1562
+ "special": true
1563
+ },
1564
+ "151838": {
1565
+ "content": "|<EXTRA_TOKENS_192>|",
1566
+ "lstrip": false,
1567
+ "normalized": false,
1568
+ "rstrip": false,
1569
+ "single_word": false,
1570
+ "special": true
1571
+ },
1572
+ "151839": {
1573
+ "content": "|<EXTRA_TOKENS_193>|",
1574
+ "lstrip": false,
1575
+ "normalized": false,
1576
+ "rstrip": false,
1577
+ "single_word": false,
1578
+ "special": true
1579
+ },
1580
+ "151840": {
1581
+ "content": "|<EXTRA_TOKENS_194>|",
1582
+ "lstrip": false,
1583
+ "normalized": false,
1584
+ "rstrip": false,
1585
+ "single_word": false,
1586
+ "special": true
1587
+ },
1588
+ "151841": {
1589
+ "content": "|<EXTRA_TOKENS_195>|",
1590
+ "lstrip": false,
1591
+ "normalized": false,
1592
+ "rstrip": false,
1593
+ "single_word": false,
1594
+ "special": true
1595
+ },
1596
+ "151842": {
1597
+ "content": "|<EXTRA_TOKENS_196>|",
1598
+ "lstrip": false,
1599
+ "normalized": false,
1600
+ "rstrip": false,
1601
+ "single_word": false,
1602
+ "special": true
1603
+ },
1604
+ "151843": {
1605
+ "content": "|<EXTRA_TOKENS_197>|",
1606
+ "lstrip": false,
1607
+ "normalized": false,
1608
+ "rstrip": false,
1609
+ "single_word": false,
1610
+ "special": true
1611
+ },
1612
+ "151844": {
1613
+ "content": "|<EXTRA_TOKENS_198>|",
1614
+ "lstrip": false,
1615
+ "normalized": false,
1616
+ "rstrip": false,
1617
+ "single_word": false,
1618
+ "special": true
1619
+ },
1620
+ "151845": {
1621
+ "content": "|<EXTRA_TOKENS_199>|",
1622
+ "lstrip": false,
1623
+ "normalized": false,
1624
+ "rstrip": false,
1625
+ "single_word": false,
1626
+ "special": true
1627
+ },
1628
+ "151846": {
1629
+ "content": "|<EXTRA_TOKENS_200>|",
1630
+ "lstrip": false,
1631
+ "normalized": false,
1632
+ "rstrip": false,
1633
+ "single_word": false,
1634
+ "special": true
1635
+ },
1636
+ "151847": {
1637
+ "content": "|<EXTRA_TOKENS_201>|",
1638
+ "lstrip": false,
1639
+ "normalized": false,
1640
+ "rstrip": false,
1641
+ "single_word": false,
1642
+ "special": true
1643
+ },
1644
+ "151848": {
1645
+ "content": "|<EXTRA_TOKENS_202>|",
1646
+ "lstrip": false,
1647
+ "normalized": false,
1648
+ "rstrip": false,
1649
+ "single_word": false,
1650
+ "special": true
1651
+ },
1652
+ "151849": {
1653
+ "content": "|<EXTRA_TOKENS_203>|",
1654
+ "lstrip": false,
1655
+ "normalized": false,
1656
+ "rstrip": false,
1657
+ "single_word": false,
1658
+ "special": true
1659
+ },
1660
+ "151850": {
1661
+ "content": "|<EXTRA_TOKENS_204>|",
1662
+ "lstrip": false,
1663
+ "normalized": false,
1664
+ "rstrip": false,
1665
+ "single_word": false,
1666
+ "special": true
1667
+ },
1668
+ "151851": {
1669
+ "content": "|<EXTRA_TOKENS_205>|",
1670
+ "lstrip": false,
1671
+ "normalized": false,
1672
+ "rstrip": false,
1673
+ "single_word": false,
1674
+ "special": true
1675
+ },
1676
+ "151852": {
1677
+ "content": "|<EXTRA_TOKENS_206>|",
1678
+ "lstrip": false,
1679
+ "normalized": false,
1680
+ "rstrip": false,
1681
+ "single_word": false,
1682
+ "special": true
1683
+ },
1684
+ "151853": {
1685
+ "content": "|<EXTRA_TOKENS_207>|",
1686
+ "lstrip": false,
1687
+ "normalized": false,
1688
+ "rstrip": false,
1689
+ "single_word": false,
1690
+ "special": true
1691
+ },
1692
+ "151854": {
1693
+ "content": "|<EXTRA_TOKENS_208>|",
1694
+ "lstrip": false,
1695
+ "normalized": false,
1696
+ "rstrip": false,
1697
+ "single_word": false,
1698
+ "special": true
1699
+ },
1700
+ "151855": {
1701
+ "content": "|<EXTRA_TOKENS_209>|",
1702
+ "lstrip": false,
1703
+ "normalized": false,
1704
+ "rstrip": false,
1705
+ "single_word": false,
1706
+ "special": true
1707
+ },
1708
+ "151856": {
1709
+ "content": "|<EXTRA_TOKENS_210>|",
1710
+ "lstrip": false,
1711
+ "normalized": false,
1712
+ "rstrip": false,
1713
+ "single_word": false,
1714
+ "special": true
1715
+ },
1716
+ "151857": {
1717
+ "content": "|<EXTRA_TOKENS_211>|",
1718
+ "lstrip": false,
1719
+ "normalized": false,
1720
+ "rstrip": false,
1721
+ "single_word": false,
1722
+ "special": true
1723
+ },
1724
+ "151858": {
1725
+ "content": "|<EXTRA_TOKENS_212>|",
1726
+ "lstrip": false,
1727
+ "normalized": false,
1728
+ "rstrip": false,
1729
+ "single_word": false,
1730
+ "special": true
1731
+ },
1732
+ "151859": {
1733
+ "content": "|<EXTRA_TOKENS_213>|",
1734
+ "lstrip": false,
1735
+ "normalized": false,
1736
+ "rstrip": false,
1737
+ "single_word": false,
1738
+ "special": true
1739
+ },
1740
+ "151860": {
1741
+ "content": "|<EXTRA_TOKENS_214>|",
1742
+ "lstrip": false,
1743
+ "normalized": false,
1744
+ "rstrip": false,
1745
+ "single_word": false,
1746
+ "special": true
1747
+ },
1748
+ "151861": {
1749
+ "content": "|<EXTRA_TOKENS_215>|",
1750
+ "lstrip": false,
1751
+ "normalized": false,
1752
+ "rstrip": false,
1753
+ "single_word": false,
1754
+ "special": true
1755
+ },
1756
+ "151862": {
1757
+ "content": "|<EXTRA_TOKENS_216>|",
1758
+ "lstrip": false,
1759
+ "normalized": false,
1760
+ "rstrip": false,
1761
+ "single_word": false,
1762
+ "special": true
1763
+ },
1764
+ "151863": {
1765
+ "content": "|<EXTRA_TOKENS_217>|",
1766
+ "lstrip": false,
1767
+ "normalized": false,
1768
+ "rstrip": false,
1769
+ "single_word": false,
1770
+ "special": true
1771
+ },
1772
+ "151864": {
1773
+ "content": "|<EXTRA_TOKENS_218>|",
1774
+ "lstrip": false,
1775
+ "normalized": false,
1776
+ "rstrip": false,
1777
+ "single_word": false,
1778
+ "special": true
1779
+ },
1780
+ "151865": {
1781
+ "content": "|<EXTRA_TOKENS_219>|",
1782
+ "lstrip": false,
1783
+ "normalized": false,
1784
+ "rstrip": false,
1785
+ "single_word": false,
1786
+ "special": true
1787
+ },
1788
+ "151866": {
1789
+ "content": "|<EXTRA_TOKENS_220>|",
1790
+ "lstrip": false,
1791
+ "normalized": false,
1792
+ "rstrip": false,
1793
+ "single_word": false,
1794
+ "special": true
1795
+ },
1796
+ "151867": {
1797
+ "content": "|<EXTRA_TOKENS_221>|",
1798
+ "lstrip": false,
1799
+ "normalized": false,
1800
+ "rstrip": false,
1801
+ "single_word": false,
1802
+ "special": true
1803
+ },
1804
+ "151868": {
1805
+ "content": "|<EXTRA_TOKENS_222>|",
1806
+ "lstrip": false,
1807
+ "normalized": false,
1808
+ "rstrip": false,
1809
+ "single_word": false,
1810
+ "special": true
1811
+ },
1812
+ "151869": {
1813
+ "content": "|<EXTRA_TOKENS_223>|",
1814
+ "lstrip": false,
1815
+ "normalized": false,
1816
+ "rstrip": false,
1817
+ "single_word": false,
1818
+ "special": true
1819
+ },
1820
+ "151870": {
1821
+ "content": "|<EXTRA_TOKENS_224>|",
1822
+ "lstrip": false,
1823
+ "normalized": false,
1824
+ "rstrip": false,
1825
+ "single_word": false,
1826
+ "special": true
1827
+ },
1828
+ "151871": {
1829
+ "content": "|<EXTRA_TOKENS_225>|",
1830
+ "lstrip": false,
1831
+ "normalized": false,
1832
+ "rstrip": false,
1833
+ "single_word": false,
1834
+ "special": true
1835
+ },
1836
+ "151872": {
1837
+ "content": "|<EXTRA_TOKENS_226>|",
1838
+ "lstrip": false,
1839
+ "normalized": false,
1840
+ "rstrip": false,
1841
+ "single_word": false,
1842
+ "special": true
1843
+ },
1844
+ "151873": {
1845
+ "content": "|<EXTRA_TOKENS_227>|",
1846
+ "lstrip": false,
1847
+ "normalized": false,
1848
+ "rstrip": false,
1849
+ "single_word": false,
1850
+ "special": true
1851
+ },
1852
+ "151874": {
1853
+ "content": "|<EXTRA_TOKENS_228>|",
1854
+ "lstrip": false,
1855
+ "normalized": false,
1856
+ "rstrip": false,
1857
+ "single_word": false,
1858
+ "special": true
1859
+ },
1860
+ "151875": {
1861
+ "content": "|<EXTRA_TOKENS_229>|",
1862
+ "lstrip": false,
1863
+ "normalized": false,
1864
+ "rstrip": false,
1865
+ "single_word": false,
1866
+ "special": true
1867
+ },
1868
+ "151876": {
1869
+ "content": "|<EXTRA_TOKENS_230>|",
1870
+ "lstrip": false,
1871
+ "normalized": false,
1872
+ "rstrip": false,
1873
+ "single_word": false,
1874
+ "special": true
1875
+ },
1876
+ "151877": {
1877
+ "content": "|<EXTRA_TOKENS_231>|",
1878
+ "lstrip": false,
1879
+ "normalized": false,
1880
+ "rstrip": false,
1881
+ "single_word": false,
1882
+ "special": true
1883
+ },
1884
+ "151878": {
1885
+ "content": "|<EXTRA_TOKENS_232>|",
1886
+ "lstrip": false,
1887
+ "normalized": false,
1888
+ "rstrip": false,
1889
+ "single_word": false,
1890
+ "special": true
1891
+ },
1892
+ "151879": {
1893
+ "content": "|<EXTRA_TOKENS_233>|",
1894
+ "lstrip": false,
1895
+ "normalized": false,
1896
+ "rstrip": false,
1897
+ "single_word": false,
1898
+ "special": true
1899
+ },
1900
+ "151880": {
1901
+ "content": "|<EXTRA_TOKENS_234>|",
1902
+ "lstrip": false,
1903
+ "normalized": false,
1904
+ "rstrip": false,
1905
+ "single_word": false,
1906
+ "special": true
1907
+ },
1908
+ "151881": {
1909
+ "content": "|<EXTRA_TOKENS_235>|",
1910
+ "lstrip": false,
1911
+ "normalized": false,
1912
+ "rstrip": false,
1913
+ "single_word": false,
1914
+ "special": true
1915
+ },
1916
+ "151882": {
1917
+ "content": "|<EXTRA_TOKENS_236>|",
1918
+ "lstrip": false,
1919
+ "normalized": false,
1920
+ "rstrip": false,
1921
+ "single_word": false,
1922
+ "special": true
1923
+ },
1924
+ "151883": {
1925
+ "content": "|<EXTRA_TOKENS_237>|",
1926
+ "lstrip": false,
1927
+ "normalized": false,
1928
+ "rstrip": false,
1929
+ "single_word": false,
1930
+ "special": true
1931
+ },
1932
+ "151884": {
1933
+ "content": "|<EXTRA_TOKENS_238>|",
1934
+ "lstrip": false,
1935
+ "normalized": false,
1936
+ "rstrip": false,
1937
+ "single_word": false,
1938
+ "special": true
1939
+ },
1940
+ "151885": {
1941
+ "content": "|<EXTRA_TOKENS_239>|",
1942
+ "lstrip": false,
1943
+ "normalized": false,
1944
+ "rstrip": false,
1945
+ "single_word": false,
1946
+ "special": true
1947
+ },
1948
+ "151886": {
1949
+ "content": "|<EXTRA_TOKENS_240>|",
1950
+ "lstrip": false,
1951
+ "normalized": false,
1952
+ "rstrip": false,
1953
+ "single_word": false,
1954
+ "special": true
1955
+ },
1956
+ "151887": {
1957
+ "content": "|<EXTRA_TOKENS_241>|",
1958
+ "lstrip": false,
1959
+ "normalized": false,
1960
+ "rstrip": false,
1961
+ "single_word": false,
1962
+ "special": true
1963
+ },
1964
+ "151888": {
1965
+ "content": "|<EXTRA_TOKENS_242>|",
1966
+ "lstrip": false,
1967
+ "normalized": false,
1968
+ "rstrip": false,
1969
+ "single_word": false,
1970
+ "special": true
1971
+ },
1972
+ "151889": {
1973
+ "content": "|<EXTRA_TOKENS_243>|",
1974
+ "lstrip": false,
1975
+ "normalized": false,
1976
+ "rstrip": false,
1977
+ "single_word": false,
1978
+ "special": true
1979
+ },
1980
+ "151890": {
1981
+ "content": "|<EXTRA_TOKENS_244>|",
1982
+ "lstrip": false,
1983
+ "normalized": false,
1984
+ "rstrip": false,
1985
+ "single_word": false,
1986
+ "special": true
1987
+ },
1988
+ "151891": {
1989
+ "content": "|<EXTRA_TOKENS_245>|",
1990
+ "lstrip": false,
1991
+ "normalized": false,
1992
+ "rstrip": false,
1993
+ "single_word": false,
1994
+ "special": true
1995
+ },
1996
+ "151892": {
1997
+ "content": "|<EXTRA_TOKENS_246>|",
1998
+ "lstrip": false,
1999
+ "normalized": false,
2000
+ "rstrip": false,
2001
+ "single_word": false,
2002
+ "special": true
2003
+ },
2004
+ "151893": {
2005
+ "content": "|<EXTRA_TOKENS_247>|",
2006
+ "lstrip": false,
2007
+ "normalized": false,
2008
+ "rstrip": false,
2009
+ "single_word": false,
2010
+ "special": true
2011
+ },
2012
+ "151894": {
2013
+ "content": "|<EXTRA_TOKENS_248>|",
2014
+ "lstrip": false,
2015
+ "normalized": false,
2016
+ "rstrip": false,
2017
+ "single_word": false,
2018
+ "special": true
2019
+ },
2020
+ "151895": {
2021
+ "content": "|<EXTRA_TOKENS_249>|",
2022
+ "lstrip": false,
2023
+ "normalized": false,
2024
+ "rstrip": false,
2025
+ "single_word": false,
2026
+ "special": true
2027
+ },
2028
+ "151896": {
2029
+ "content": "|<EXTRA_TOKENS_250>|",
2030
+ "lstrip": false,
2031
+ "normalized": false,
2032
+ "rstrip": false,
2033
+ "single_word": false,
2034
+ "special": true
2035
+ },
2036
+ "151897": {
2037
+ "content": "|<EXTRA_TOKENS_251>|",
2038
+ "lstrip": false,
2039
+ "normalized": false,
2040
+ "rstrip": false,
2041
+ "single_word": false,
2042
+ "special": true
2043
+ },
2044
+ "151898": {
2045
+ "content": "|<EXTRA_TOKENS_252>|",
2046
+ "lstrip": false,
2047
+ "normalized": false,
2048
+ "rstrip": false,
2049
+ "single_word": false,
2050
+ "special": true
2051
+ },
2052
+ "151899": {
2053
+ "content": "|<EXTRA_TOKENS_253>|",
2054
+ "lstrip": false,
2055
+ "normalized": false,
2056
+ "rstrip": false,
2057
+ "single_word": false,
2058
+ "special": true
2059
+ },
2060
+ "151900": {
2061
+ "content": "|<EXTRA_TOKENS_254>|",
2062
+ "lstrip": false,
2063
+ "normalized": false,
2064
+ "rstrip": false,
2065
+ "single_word": false,
2066
+ "special": true
2067
+ },
2068
+ "151901": {
2069
+ "content": "|<EXTRA_TOKENS_255>|",
2070
+ "lstrip": false,
2071
+ "normalized": false,
2072
+ "rstrip": false,
2073
+ "single_word": false,
2074
+ "special": true
2075
+ },
2076
+ "151902": {
2077
+ "content": "|<EXTRA_TOKENS_256>|",
2078
+ "lstrip": false,
2079
+ "normalized": false,
2080
+ "rstrip": false,
2081
+ "single_word": false,
2082
+ "special": true
2083
+ },
2084
+ "151903": {
2085
+ "content": "|<EXTRA_TOKENS_257>|",
2086
+ "lstrip": false,
2087
+ "normalized": false,
2088
+ "rstrip": false,
2089
+ "single_word": false,
2090
+ "special": true
2091
+ },
2092
+ "151904": {
2093
+ "content": "|<EXTRA_TOKENS_258>|",
2094
+ "lstrip": false,
2095
+ "normalized": false,
2096
+ "rstrip": false,
2097
+ "single_word": false,
2098
+ "special": true
2099
+ },
2100
+ "151905": {
2101
+ "content": "|<EXTRA_TOKENS_259>|",
2102
+ "lstrip": false,
2103
+ "normalized": false,
2104
+ "rstrip": false,
2105
+ "single_word": false,
2106
+ "special": true
2107
+ },
2108
+ "151906": {
2109
+ "content": "|<EXTRA_TOKENS_260>|",
2110
+ "lstrip": false,
2111
+ "normalized": false,
2112
+ "rstrip": false,
2113
+ "single_word": false,
2114
+ "special": true
2115
+ },
2116
+ "151907": {
2117
+ "content": "|<EXTRA_TOKENS_261>|",
2118
+ "lstrip": false,
2119
+ "normalized": false,
2120
+ "rstrip": false,
2121
+ "single_word": false,
2122
+ "special": true
2123
+ },
2124
+ "151908": {
2125
+ "content": "|<EXTRA_TOKENS_262>|",
2126
+ "lstrip": false,
2127
+ "normalized": false,
2128
+ "rstrip": false,
2129
+ "single_word": false,
2130
+ "special": true
2131
+ },
2132
+ "151909": {
2133
+ "content": "|<EXTRA_TOKENS_263>|",
2134
+ "lstrip": false,
2135
+ "normalized": false,
2136
+ "rstrip": false,
2137
+ "single_word": false,
2138
+ "special": true
2139
+ },
2140
+ "151910": {
2141
+ "content": "|<EXTRA_TOKENS_264>|",
2142
+ "lstrip": false,
2143
+ "normalized": false,
2144
+ "rstrip": false,
2145
+ "single_word": false,
2146
+ "special": true
2147
+ },
2148
+ "151911": {
2149
+ "content": "|<EXTRA_TOKENS_265>|",
2150
+ "lstrip": false,
2151
+ "normalized": false,
2152
+ "rstrip": false,
2153
+ "single_word": false,
2154
+ "special": true
2155
+ },
2156
+ "151912": {
2157
+ "content": "|<EXTRA_TOKENS_266>|",
2158
+ "lstrip": false,
2159
+ "normalized": false,
2160
+ "rstrip": false,
2161
+ "single_word": false,
2162
+ "special": true
2163
+ },
2164
+ "151913": {
2165
+ "content": "|<EXTRA_TOKENS_267>|",
2166
+ "lstrip": false,
2167
+ "normalized": false,
2168
+ "rstrip": false,
2169
+ "single_word": false,
2170
+ "special": true
2171
+ },
2172
+ "151914": {
2173
+ "content": "|<EXTRA_TOKENS_268>|",
2174
+ "lstrip": false,
2175
+ "normalized": false,
2176
+ "rstrip": false,
2177
+ "single_word": false,
2178
+ "special": true
2179
+ },
2180
+ "151915": {
2181
+ "content": "|<EXTRA_TOKENS_269>|",
2182
+ "lstrip": false,
2183
+ "normalized": false,
2184
+ "rstrip": false,
2185
+ "single_word": false,
2186
+ "special": true
2187
+ },
2188
+ "151916": {
2189
+ "content": "|<EXTRA_TOKENS_270>|",
2190
+ "lstrip": false,
2191
+ "normalized": false,
2192
+ "rstrip": false,
2193
+ "single_word": false,
2194
+ "special": true
2195
+ },
2196
+ "151917": {
2197
+ "content": "|<EXTRA_TOKENS_271>|",
2198
+ "lstrip": false,
2199
+ "normalized": false,
2200
+ "rstrip": false,
2201
+ "single_word": false,
2202
+ "special": true
2203
+ },
2204
+ "151918": {
2205
+ "content": "|<EXTRA_TOKENS_272>|",
2206
+ "lstrip": false,
2207
+ "normalized": false,
2208
+ "rstrip": false,
2209
+ "single_word": false,
2210
+ "special": true
2211
+ },
2212
+ "151919": {
2213
+ "content": "|<EXTRA_TOKENS_273>|",
2214
+ "lstrip": false,
2215
+ "normalized": false,
2216
+ "rstrip": false,
2217
+ "single_word": false,
2218
+ "special": true
2219
+ },
2220
+ "151920": {
2221
+ "content": "|<EXTRA_TOKENS_274>|",
2222
+ "lstrip": false,
2223
+ "normalized": false,
2224
+ "rstrip": false,
2225
+ "single_word": false,
2226
+ "special": true
2227
+ },
2228
+ "151921": {
2229
+ "content": "|<EXTRA_TOKENS_275>|",
2230
+ "lstrip": false,
2231
+ "normalized": false,
2232
+ "rstrip": false,
2233
+ "single_word": false,
2234
+ "special": true
2235
+ },
2236
+ "151922": {
2237
+ "content": "|<EXTRA_TOKENS_276>|",
2238
+ "lstrip": false,
2239
+ "normalized": false,
2240
+ "rstrip": false,
2241
+ "single_word": false,
2242
+ "special": true
2243
+ },
2244
+ "151923": {
2245
+ "content": "|<EXTRA_TOKENS_277>|",
2246
+ "lstrip": false,
2247
+ "normalized": false,
2248
+ "rstrip": false,
2249
+ "single_word": false,
2250
+ "special": true
2251
+ },
2252
+ "151924": {
2253
+ "content": "|<EXTRA_TOKENS_278>|",
2254
+ "lstrip": false,
2255
+ "normalized": false,
2256
+ "rstrip": false,
2257
+ "single_word": false,
2258
+ "special": true
2259
+ },
2260
+ "151925": {
2261
+ "content": "|<EXTRA_TOKENS_279>|",
2262
+ "lstrip": false,
2263
+ "normalized": false,
2264
+ "rstrip": false,
2265
+ "single_word": false,
2266
+ "special": true
2267
+ },
2268
+ "151926": {
2269
+ "content": "|<EXTRA_TOKENS_280>|",
2270
+ "lstrip": false,
2271
+ "normalized": false,
2272
+ "rstrip": false,
2273
+ "single_word": false,
2274
+ "special": true
2275
+ },
2276
+ "151927": {
2277
+ "content": "|<EXTRA_TOKENS_281>|",
2278
+ "lstrip": false,
2279
+ "normalized": false,
2280
+ "rstrip": false,
2281
+ "single_word": false,
2282
+ "special": true
2283
+ },
2284
+ "151928": {
2285
+ "content": "|<EXTRA_TOKENS_282>|",
2286
+ "lstrip": false,
2287
+ "normalized": false,
2288
+ "rstrip": false,
2289
+ "single_word": false,
2290
+ "special": true
2291
+ },
2292
+ "151929": {
2293
+ "content": "|<EXTRA_TOKENS_283>|",
2294
+ "lstrip": false,
2295
+ "normalized": false,
2296
+ "rstrip": false,
2297
+ "single_word": false,
2298
+ "special": true
2299
+ },
2300
+ "151930": {
2301
+ "content": "|<EXTRA_TOKENS_284>|",
2302
+ "lstrip": false,
2303
+ "normalized": false,
2304
+ "rstrip": false,
2305
+ "single_word": false,
2306
+ "special": true
2307
+ },
2308
+ "151931": {
2309
+ "content": "|<EXTRA_TOKENS_285>|",
2310
+ "lstrip": false,
2311
+ "normalized": false,
2312
+ "rstrip": false,
2313
+ "single_word": false,
2314
+ "special": true
2315
+ },
2316
+ "151932": {
2317
+ "content": "|<EXTRA_TOKENS_286>|",
2318
+ "lstrip": false,
2319
+ "normalized": false,
2320
+ "rstrip": false,
2321
+ "single_word": false,
2322
+ "special": true
2323
+ },
2324
+ "151933": {
2325
+ "content": "|<EXTRA_TOKENS_287>|",
2326
+ "lstrip": false,
2327
+ "normalized": false,
2328
+ "rstrip": false,
2329
+ "single_word": false,
2330
+ "special": true
2331
+ },
2332
+ "151934": {
2333
+ "content": "|<EXTRA_TOKENS_288>|",
2334
+ "lstrip": false,
2335
+ "normalized": false,
2336
+ "rstrip": false,
2337
+ "single_word": false,
2338
+ "special": true
2339
+ },
2340
+ "151935": {
2341
+ "content": "|<EXTRA_TOKENS_289>|",
2342
+ "lstrip": false,
2343
+ "normalized": false,
2344
+ "rstrip": false,
2345
+ "single_word": false,
2346
+ "special": true
2347
+ },
2348
+ "151936": {
2349
+ "content": "|<EXTRA_TOKENS_290>|",
2350
+ "lstrip": false,
2351
+ "normalized": false,
2352
+ "rstrip": false,
2353
+ "single_word": false,
2354
+ "special": true
2355
+ },
2356
+ "151937": {
2357
+ "content": "|<EXTRA_TOKENS_291>|",
2358
+ "lstrip": false,
2359
+ "normalized": false,
2360
+ "rstrip": false,
2361
+ "single_word": false,
2362
+ "special": true
2363
+ },
2364
+ "151938": {
2365
+ "content": "|<EXTRA_TOKENS_292>|",
2366
+ "lstrip": false,
2367
+ "normalized": false,
2368
+ "rstrip": false,
2369
+ "single_word": false,
2370
+ "special": true
2371
+ },
2372
+ "151939": {
2373
+ "content": "|<EXTRA_TOKENS_293>|",
2374
+ "lstrip": false,
2375
+ "normalized": false,
2376
+ "rstrip": false,
2377
+ "single_word": false,
2378
+ "special": true
2379
+ },
2380
+ "151940": {
2381
+ "content": "|<EXTRA_TOKENS_294>|",
2382
+ "lstrip": false,
2383
+ "normalized": false,
2384
+ "rstrip": false,
2385
+ "single_word": false,
2386
+ "special": true
2387
+ },
2388
+ "151941": {
2389
+ "content": "|<EXTRA_TOKENS_295>|",
2390
+ "lstrip": false,
2391
+ "normalized": false,
2392
+ "rstrip": false,
2393
+ "single_word": false,
2394
+ "special": true
2395
+ },
2396
+ "151942": {
2397
+ "content": "|<EXTRA_TOKENS_296>|",
2398
+ "lstrip": false,
2399
+ "normalized": false,
2400
+ "rstrip": false,
2401
+ "single_word": false,
2402
+ "special": true
2403
+ },
2404
+ "151943": {
2405
+ "content": "|<EXTRA_TOKENS_297>|",
2406
+ "lstrip": false,
2407
+ "normalized": false,
2408
+ "rstrip": false,
2409
+ "single_word": false,
2410
+ "special": true
2411
+ },
2412
+ "151944": {
2413
+ "content": "|<EXTRA_TOKENS_298>|",
2414
+ "lstrip": false,
2415
+ "normalized": false,
2416
+ "rstrip": false,
2417
+ "single_word": false,
2418
+ "special": true
2419
+ },
2420
+ "151945": {
2421
+ "content": "|<EXTRA_TOKENS_299>|",
2422
+ "lstrip": false,
2423
+ "normalized": false,
2424
+ "rstrip": false,
2425
+ "single_word": false,
2426
+ "special": true
2427
+ },
2428
+ "151946": {
2429
+ "content": "|<EXTRA_TOKENS_300>|",
2430
+ "lstrip": false,
2431
+ "normalized": false,
2432
+ "rstrip": false,
2433
+ "single_word": false,
2434
+ "special": true
2435
+ },
2436
+ "151947": {
2437
+ "content": "|<EXTRA_TOKENS_301>|",
2438
+ "lstrip": false,
2439
+ "normalized": false,
2440
+ "rstrip": false,
2441
+ "single_word": false,
2442
+ "special": true
2443
+ },
2444
+ "151948": {
2445
+ "content": "|<EXTRA_TOKENS_302>|",
2446
+ "lstrip": false,
2447
+ "normalized": false,
2448
+ "rstrip": false,
2449
+ "single_word": false,
2450
+ "special": true
2451
+ },
2452
+ "151949": {
2453
+ "content": "|<EXTRA_TOKENS_303>|",
2454
+ "lstrip": false,
2455
+ "normalized": false,
2456
+ "rstrip": false,
2457
+ "single_word": false,
2458
+ "special": true
2459
+ },
2460
+ "151950": {
2461
+ "content": "|<EXTRA_TOKENS_304>|",
2462
+ "lstrip": false,
2463
+ "normalized": false,
2464
+ "rstrip": false,
2465
+ "single_word": false,
2466
+ "special": true
2467
+ },
2468
+ "151951": {
2469
+ "content": "|<EXTRA_TOKENS_305>|",
2470
+ "lstrip": false,
2471
+ "normalized": false,
2472
+ "rstrip": false,
2473
+ "single_word": false,
2474
+ "special": true
2475
+ },
2476
+ "151952": {
2477
+ "content": "|<EXTRA_TOKENS_306>|",
2478
+ "lstrip": false,
2479
+ "normalized": false,
2480
+ "rstrip": false,
2481
+ "single_word": false,
2482
+ "special": true
2483
+ },
2484
+ "151953": {
2485
+ "content": "|<EXTRA_TOKENS_307>|",
2486
+ "lstrip": false,
2487
+ "normalized": false,
2488
+ "rstrip": false,
2489
+ "single_word": false,
2490
+ "special": true
2491
+ },
2492
+ "151954": {
2493
+ "content": "|<EXTRA_TOKENS_308>|",
2494
+ "lstrip": false,
2495
+ "normalized": false,
2496
+ "rstrip": false,
2497
+ "single_word": false,
2498
+ "special": true
2499
+ },
2500
+ "151955": {
2501
+ "content": "|<EXTRA_TOKENS_309>|",
2502
+ "lstrip": false,
2503
+ "normalized": false,
2504
+ "rstrip": false,
2505
+ "single_word": false,
2506
+ "special": true
2507
+ },
2508
+ "151956": {
2509
+ "content": "|<EXTRA_TOKENS_310>|",
2510
+ "lstrip": false,
2511
+ "normalized": false,
2512
+ "rstrip": false,
2513
+ "single_word": false,
2514
+ "special": true
2515
+ },
2516
+ "151957": {
2517
+ "content": "|<EXTRA_TOKENS_311>|",
2518
+ "lstrip": false,
2519
+ "normalized": false,
2520
+ "rstrip": false,
2521
+ "single_word": false,
2522
+ "special": true
2523
+ },
2524
+ "151958": {
2525
+ "content": "|<EXTRA_TOKENS_312>|",
2526
+ "lstrip": false,
2527
+ "normalized": false,
2528
+ "rstrip": false,
2529
+ "single_word": false,
2530
+ "special": true
2531
+ },
2532
+ "151959": {
2533
+ "content": "|<EXTRA_TOKENS_313>|",
2534
+ "lstrip": false,
2535
+ "normalized": false,
2536
+ "rstrip": false,
2537
+ "single_word": false,
2538
+ "special": true
2539
+ },
2540
+ "151960": {
2541
+ "content": "|<EXTRA_TOKENS_314>|",
2542
+ "lstrip": false,
2543
+ "normalized": false,
2544
+ "rstrip": false,
2545
+ "single_word": false,
2546
+ "special": true
2547
+ },
2548
+ "151961": {
2549
+ "content": "|<EXTRA_TOKENS_315>|",
2550
+ "lstrip": false,
2551
+ "normalized": false,
2552
+ "rstrip": false,
2553
+ "single_word": false,
2554
+ "special": true
2555
+ },
2556
+ "151962": {
2557
+ "content": "|<EXTRA_TOKENS_316>|",
2558
+ "lstrip": false,
2559
+ "normalized": false,
2560
+ "rstrip": false,
2561
+ "single_word": false,
2562
+ "special": true
2563
+ },
2564
+ "151963": {
2565
+ "content": "|<EXTRA_TOKENS_317>|",
2566
+ "lstrip": false,
2567
+ "normalized": false,
2568
+ "rstrip": false,
2569
+ "single_word": false,
2570
+ "special": true
2571
+ },
2572
+ "151964": {
2573
+ "content": "|<EXTRA_TOKENS_318>|",
2574
+ "lstrip": false,
2575
+ "normalized": false,
2576
+ "rstrip": false,
2577
+ "single_word": false,
2578
+ "special": true
2579
+ },
2580
+ "151965": {
2581
+ "content": "|<EXTRA_TOKENS_319>|",
2582
+ "lstrip": false,
2583
+ "normalized": false,
2584
+ "rstrip": false,
2585
+ "single_word": false,
2586
+ "special": true
2587
+ },
2588
+ "151966": {
2589
+ "content": "|<EXTRA_TOKENS_320>|",
2590
+ "lstrip": false,
2591
+ "normalized": false,
2592
+ "rstrip": false,
2593
+ "single_word": false,
2594
+ "special": true
2595
+ },
2596
+ "151967": {
2597
+ "content": "|<EXTRA_TOKENS_321>|",
2598
+ "lstrip": false,
2599
+ "normalized": false,
2600
+ "rstrip": false,
2601
+ "single_word": false,
2602
+ "special": true
2603
+ },
2604
+ "151968": {
2605
+ "content": "|<EXTRA_TOKENS_322>|",
2606
+ "lstrip": false,
2607
+ "normalized": false,
2608
+ "rstrip": false,
2609
+ "single_word": false,
2610
+ "special": true
2611
+ },
2612
+ "151969": {
2613
+ "content": "|<EXTRA_TOKENS_323>|",
2614
+ "lstrip": false,
2615
+ "normalized": false,
2616
+ "rstrip": false,
2617
+ "single_word": false,
2618
+ "special": true
2619
+ },
2620
+ "151970": {
2621
+ "content": "|<EXTRA_TOKENS_324>|",
2622
+ "lstrip": false,
2623
+ "normalized": false,
2624
+ "rstrip": false,
2625
+ "single_word": false,
2626
+ "special": true
2627
+ },
2628
+ "151971": {
2629
+ "content": "|<EXTRA_TOKENS_325>|",
2630
+ "lstrip": false,
2631
+ "normalized": false,
2632
+ "rstrip": false,
2633
+ "single_word": false,
2634
+ "special": true
2635
+ },
2636
+ "151972": {
2637
+ "content": "|<EXTRA_TOKENS_326>|",
2638
+ "lstrip": false,
2639
+ "normalized": false,
2640
+ "rstrip": false,
2641
+ "single_word": false,
2642
+ "special": true
2643
+ },
2644
+ "151973": {
2645
+ "content": "|<EXTRA_TOKENS_327>|",
2646
+ "lstrip": false,
2647
+ "normalized": false,
2648
+ "rstrip": false,
2649
+ "single_word": false,
2650
+ "special": true
2651
+ },
2652
+ "151974": {
2653
+ "content": "|<EXTRA_TOKENS_328>|",
2654
+ "lstrip": false,
2655
+ "normalized": false,
2656
+ "rstrip": false,
2657
+ "single_word": false,
2658
+ "special": true
2659
+ },
2660
+ "151975": {
2661
+ "content": "|<EXTRA_TOKENS_329>|",
2662
+ "lstrip": false,
2663
+ "normalized": false,
2664
+ "rstrip": false,
2665
+ "single_word": false,
2666
+ "special": true
2667
+ },
2668
+ "151976": {
2669
+ "content": "|<EXTRA_TOKENS_330>|",
2670
+ "lstrip": false,
2671
+ "normalized": false,
2672
+ "rstrip": false,
2673
+ "single_word": false,
2674
+ "special": true
2675
+ },
2676
+ "151977": {
2677
+ "content": "|<EXTRA_TOKENS_331>|",
2678
+ "lstrip": false,
2679
+ "normalized": false,
2680
+ "rstrip": false,
2681
+ "single_word": false,
2682
+ "special": true
2683
+ },
2684
+ "151978": {
2685
+ "content": "|<EXTRA_TOKENS_332>|",
2686
+ "lstrip": false,
2687
+ "normalized": false,
2688
+ "rstrip": false,
2689
+ "single_word": false,
2690
+ "special": true
2691
+ },
2692
+ "151979": {
2693
+ "content": "|<EXTRA_TOKENS_333>|",
2694
+ "lstrip": false,
2695
+ "normalized": false,
2696
+ "rstrip": false,
2697
+ "single_word": false,
2698
+ "special": true
2699
+ },
2700
+ "151980": {
2701
+ "content": "|<EXTRA_TOKENS_334>|",
2702
+ "lstrip": false,
2703
+ "normalized": false,
2704
+ "rstrip": false,
2705
+ "single_word": false,
2706
+ "special": true
2707
+ },
2708
+ "151981": {
2709
+ "content": "|<EXTRA_TOKENS_335>|",
2710
+ "lstrip": false,
2711
+ "normalized": false,
2712
+ "rstrip": false,
2713
+ "single_word": false,
2714
+ "special": true
2715
+ },
2716
+ "151982": {
2717
+ "content": "|<EXTRA_TOKENS_336>|",
2718
+ "lstrip": false,
2719
+ "normalized": false,
2720
+ "rstrip": false,
2721
+ "single_word": false,
2722
+ "special": true
2723
+ },
2724
+ "151983": {
2725
+ "content": "|<EXTRA_TOKENS_337>|",
2726
+ "lstrip": false,
2727
+ "normalized": false,
2728
+ "rstrip": false,
2729
+ "single_word": false,
2730
+ "special": true
2731
+ },
2732
+ "151984": {
2733
+ "content": "|<EXTRA_TOKENS_338>|",
2734
+ "lstrip": false,
2735
+ "normalized": false,
2736
+ "rstrip": false,
2737
+ "single_word": false,
2738
+ "special": true
2739
+ },
2740
+ "151985": {
2741
+ "content": "|<EXTRA_TOKENS_339>|",
2742
+ "lstrip": false,
2743
+ "normalized": false,
2744
+ "rstrip": false,
2745
+ "single_word": false,
2746
+ "special": true
2747
+ },
2748
+ "151986": {
2749
+ "content": "|<EXTRA_TOKENS_340>|",
2750
+ "lstrip": false,
2751
+ "normalized": false,
2752
+ "rstrip": false,
2753
+ "single_word": false,
2754
+ "special": true
2755
+ },
2756
+ "151987": {
2757
+ "content": "|<EXTRA_TOKENS_341>|",
2758
+ "lstrip": false,
2759
+ "normalized": false,
2760
+ "rstrip": false,
2761
+ "single_word": false,
2762
+ "special": true
2763
+ },
2764
+ "151988": {
2765
+ "content": "|<EXTRA_TOKENS_342>|",
2766
+ "lstrip": false,
2767
+ "normalized": false,
2768
+ "rstrip": false,
2769
+ "single_word": false,
2770
+ "special": true
2771
+ },
2772
+ "151989": {
2773
+ "content": "|<EXTRA_TOKENS_343>|",
2774
+ "lstrip": false,
2775
+ "normalized": false,
2776
+ "rstrip": false,
2777
+ "single_word": false,
2778
+ "special": true
2779
+ },
2780
+ "151990": {
2781
+ "content": "|<EXTRA_TOKENS_344>|",
2782
+ "lstrip": false,
2783
+ "normalized": false,
2784
+ "rstrip": false,
2785
+ "single_word": false,
2786
+ "special": true
2787
+ },
2788
+ "151991": {
2789
+ "content": "|<EXTRA_TOKENS_345>|",
2790
+ "lstrip": false,
2791
+ "normalized": false,
2792
+ "rstrip": false,
2793
+ "single_word": false,
2794
+ "special": true
2795
+ },
2796
+ "151992": {
2797
+ "content": "|<EXTRA_TOKENS_346>|",
2798
+ "lstrip": false,
2799
+ "normalized": false,
2800
+ "rstrip": false,
2801
+ "single_word": false,
2802
+ "special": true
2803
+ },
2804
+ "151993": {
2805
+ "content": "|<EXTRA_TOKENS_347>|",
2806
+ "lstrip": false,
2807
+ "normalized": false,
2808
+ "rstrip": false,
2809
+ "single_word": false,
2810
+ "special": true
2811
+ },
2812
+ "151994": {
2813
+ "content": "|<EXTRA_TOKENS_348>|",
2814
+ "lstrip": false,
2815
+ "normalized": false,
2816
+ "rstrip": false,
2817
+ "single_word": false,
2818
+ "special": true
2819
+ },
2820
+ "151995": {
2821
+ "content": "|<EXTRA_TOKENS_349>|",
2822
+ "lstrip": false,
2823
+ "normalized": false,
2824
+ "rstrip": false,
2825
+ "single_word": false,
2826
+ "special": true
2827
+ },
2828
+ "151996": {
2829
+ "content": "|<EXTRA_TOKENS_350>|",
2830
+ "lstrip": false,
2831
+ "normalized": false,
2832
+ "rstrip": false,
2833
+ "single_word": false,
2834
+ "special": true
2835
+ },
2836
+ "151997": {
2837
+ "content": "|<EXTRA_TOKENS_351>|",
2838
+ "lstrip": false,
2839
+ "normalized": false,
2840
+ "rstrip": false,
2841
+ "single_word": false,
2842
+ "special": true
2843
+ },
2844
+ "151998": {
2845
+ "content": "|<EXTRA_TOKENS_352>|",
2846
+ "lstrip": false,
2847
+ "normalized": false,
2848
+ "rstrip": false,
2849
+ "single_word": false,
2850
+ "special": true
2851
+ },
2852
+ "151999": {
2853
+ "content": "|<EXTRA_TOKENS_353>|",
2854
+ "lstrip": false,
2855
+ "normalized": false,
2856
+ "rstrip": false,
2857
+ "single_word": false,
2858
+ "special": true
2859
+ },
2860
+ "152000": {
2861
+ "content": "|<EXTRA_TOKENS_354>|",
2862
+ "lstrip": false,
2863
+ "normalized": false,
2864
+ "rstrip": false,
2865
+ "single_word": false,
2866
+ "special": true
2867
+ },
2868
+ "152001": {
2869
+ "content": "|<EXTRA_TOKENS_355>|",
2870
+ "lstrip": false,
2871
+ "normalized": false,
2872
+ "rstrip": false,
2873
+ "single_word": false,
2874
+ "special": true
2875
+ },
2876
+ "152002": {
2877
+ "content": "|<EXTRA_TOKENS_356>|",
2878
+ "lstrip": false,
2879
+ "normalized": false,
2880
+ "rstrip": false,
2881
+ "single_word": false,
2882
+ "special": true
2883
+ },
2884
+ "152003": {
2885
+ "content": "|<EXTRA_TOKENS_357>|",
2886
+ "lstrip": false,
2887
+ "normalized": false,
2888
+ "rstrip": false,
2889
+ "single_word": false,
2890
+ "special": true
2891
+ },
2892
+ "152004": {
2893
+ "content": "|<EXTRA_TOKENS_358>|",
2894
+ "lstrip": false,
2895
+ "normalized": false,
2896
+ "rstrip": false,
2897
+ "single_word": false,
2898
+ "special": true
2899
+ },
2900
+ "152005": {
2901
+ "content": "|<EXTRA_TOKENS_359>|",
2902
+ "lstrip": false,
2903
+ "normalized": false,
2904
+ "rstrip": false,
2905
+ "single_word": false,
2906
+ "special": true
2907
+ },
2908
+ "152006": {
2909
+ "content": "|<EXTRA_TOKENS_360>|",
2910
+ "lstrip": false,
2911
+ "normalized": false,
2912
+ "rstrip": false,
2913
+ "single_word": false,
2914
+ "special": true
2915
+ },
2916
+ "152007": {
2917
+ "content": "|<EXTRA_TOKENS_361>|",
2918
+ "lstrip": false,
2919
+ "normalized": false,
2920
+ "rstrip": false,
2921
+ "single_word": false,
2922
+ "special": true
2923
+ },
2924
+ "152008": {
2925
+ "content": "|<EXTRA_TOKENS_362>|",
2926
+ "lstrip": false,
2927
+ "normalized": false,
2928
+ "rstrip": false,
2929
+ "single_word": false,
2930
+ "special": true
2931
+ },
2932
+ "152009": {
2933
+ "content": "|<EXTRA_TOKENS_363>|",
2934
+ "lstrip": false,
2935
+ "normalized": false,
2936
+ "rstrip": false,
2937
+ "single_word": false,
2938
+ "special": true
2939
+ },
2940
+ "152010": {
2941
+ "content": "|<EXTRA_TOKENS_364>|",
2942
+ "lstrip": false,
2943
+ "normalized": false,
2944
+ "rstrip": false,
2945
+ "single_word": false,
2946
+ "special": true
2947
+ },
2948
+ "152011": {
2949
+ "content": "|<EXTRA_TOKENS_365>|",
2950
+ "lstrip": false,
2951
+ "normalized": false,
2952
+ "rstrip": false,
2953
+ "single_word": false,
2954
+ "special": true
2955
+ },
2956
+ "152012": {
2957
+ "content": "|<EXTRA_TOKENS_366>|",
2958
+ "lstrip": false,
2959
+ "normalized": false,
2960
+ "rstrip": false,
2961
+ "single_word": false,
2962
+ "special": true
2963
+ },
2964
+ "152013": {
2965
+ "content": "|<EXTRA_TOKENS_367>|",
2966
+ "lstrip": false,
2967
+ "normalized": false,
2968
+ "rstrip": false,
2969
+ "single_word": false,
2970
+ "special": true
2971
+ },
2972
+ "152014": {
2973
+ "content": "|<EXTRA_TOKENS_368>|",
2974
+ "lstrip": false,
2975
+ "normalized": false,
2976
+ "rstrip": false,
2977
+ "single_word": false,
2978
+ "special": true
2979
+ },
2980
+ "152015": {
2981
+ "content": "|<EXTRA_TOKENS_369>|",
2982
+ "lstrip": false,
2983
+ "normalized": false,
2984
+ "rstrip": false,
2985
+ "single_word": false,
2986
+ "special": true
2987
+ },
2988
+ "152016": {
2989
+ "content": "|<EXTRA_TOKENS_370>|",
2990
+ "lstrip": false,
2991
+ "normalized": false,
2992
+ "rstrip": false,
2993
+ "single_word": false,
2994
+ "special": true
2995
+ },
2996
+ "152017": {
2997
+ "content": "|<EXTRA_TOKENS_371>|",
2998
+ "lstrip": false,
2999
+ "normalized": false,
3000
+ "rstrip": false,
3001
+ "single_word": false,
3002
+ "special": true
3003
+ },
3004
+ "152018": {
3005
+ "content": "|<EXTRA_TOKENS_372>|",
3006
+ "lstrip": false,
3007
+ "normalized": false,
3008
+ "rstrip": false,
3009
+ "single_word": false,
3010
+ "special": true
3011
+ },
3012
+ "152019": {
3013
+ "content": "|<EXTRA_TOKENS_373>|",
3014
+ "lstrip": false,
3015
+ "normalized": false,
3016
+ "rstrip": false,
3017
+ "single_word": false,
3018
+ "special": true
3019
+ },
3020
+ "152020": {
3021
+ "content": "|<EXTRA_TOKENS_374>|",
3022
+ "lstrip": false,
3023
+ "normalized": false,
3024
+ "rstrip": false,
3025
+ "single_word": false,
3026
+ "special": true
3027
+ },
3028
+ "152021": {
3029
+ "content": "|<EXTRA_TOKENS_375>|",
3030
+ "lstrip": false,
3031
+ "normalized": false,
3032
+ "rstrip": false,
3033
+ "single_word": false,
3034
+ "special": true
3035
+ },
3036
+ "152022": {
3037
+ "content": "|<EXTRA_TOKENS_376>|",
3038
+ "lstrip": false,
3039
+ "normalized": false,
3040
+ "rstrip": false,
3041
+ "single_word": false,
3042
+ "special": true
3043
+ },
3044
+ "152023": {
3045
+ "content": "|<EXTRA_TOKENS_377>|",
3046
+ "lstrip": false,
3047
+ "normalized": false,
3048
+ "rstrip": false,
3049
+ "single_word": false,
3050
+ "special": true
3051
+ },
3052
+ "152024": {
3053
+ "content": "|<EXTRA_TOKENS_378>|",
3054
+ "lstrip": false,
3055
+ "normalized": false,
3056
+ "rstrip": false,
3057
+ "single_word": false,
3058
+ "special": true
3059
+ },
3060
+ "152025": {
3061
+ "content": "|<EXTRA_TOKENS_379>|",
3062
+ "lstrip": false,
3063
+ "normalized": false,
3064
+ "rstrip": false,
3065
+ "single_word": false,
3066
+ "special": true
3067
+ },
3068
+ "152026": {
3069
+ "content": "|<EXTRA_TOKENS_380>|",
3070
+ "lstrip": false,
3071
+ "normalized": false,
3072
+ "rstrip": false,
3073
+ "single_word": false,
3074
+ "special": true
3075
+ },
3076
+ "152027": {
3077
+ "content": "|<EXTRA_TOKENS_381>|",
3078
+ "lstrip": false,
3079
+ "normalized": false,
3080
+ "rstrip": false,
3081
+ "single_word": false,
3082
+ "special": true
3083
+ },
3084
+ "152028": {
3085
+ "content": "|<EXTRA_TOKENS_382>|",
3086
+ "lstrip": false,
3087
+ "normalized": false,
3088
+ "rstrip": false,
3089
+ "single_word": false,
3090
+ "special": true
3091
+ },
3092
+ "152029": {
3093
+ "content": "|<EXTRA_TOKENS_383>|",
3094
+ "lstrip": false,
3095
+ "normalized": false,
3096
+ "rstrip": false,
3097
+ "single_word": false,
3098
+ "special": true
3099
+ },
3100
+ "152030": {
3101
+ "content": "|<EXTRA_TOKENS_384>|",
3102
+ "lstrip": false,
3103
+ "normalized": false,
3104
+ "rstrip": false,
3105
+ "single_word": false,
3106
+ "special": true
3107
+ },
3108
+ "152031": {
3109
+ "content": "|<EXTRA_TOKENS_385>|",
3110
+ "lstrip": false,
3111
+ "normalized": false,
3112
+ "rstrip": false,
3113
+ "single_word": false,
3114
+ "special": true
3115
+ },
3116
+ "152032": {
3117
+ "content": "|<EXTRA_TOKENS_386>|",
3118
+ "lstrip": false,
3119
+ "normalized": false,
3120
+ "rstrip": false,
3121
+ "single_word": false,
3122
+ "special": true
3123
+ },
3124
+ "152033": {
3125
+ "content": "|<EXTRA_TOKENS_387>|",
3126
+ "lstrip": false,
3127
+ "normalized": false,
3128
+ "rstrip": false,
3129
+ "single_word": false,
3130
+ "special": true
3131
+ },
3132
+ "152034": {
3133
+ "content": "|<EXTRA_TOKENS_388>|",
3134
+ "lstrip": false,
3135
+ "normalized": false,
3136
+ "rstrip": false,
3137
+ "single_word": false,
3138
+ "special": true
3139
+ },
3140
+ "152035": {
3141
+ "content": "|<EXTRA_TOKENS_389>|",
3142
+ "lstrip": false,
3143
+ "normalized": false,
3144
+ "rstrip": false,
3145
+ "single_word": false,
3146
+ "special": true
3147
+ },
3148
+ "152036": {
3149
+ "content": "|<EXTRA_TOKENS_390>|",
3150
+ "lstrip": false,
3151
+ "normalized": false,
3152
+ "rstrip": false,
3153
+ "single_word": false,
3154
+ "special": true
3155
+ },
3156
+ "152037": {
3157
+ "content": "|<EXTRA_TOKENS_391>|",
3158
+ "lstrip": false,
3159
+ "normalized": false,
3160
+ "rstrip": false,
3161
+ "single_word": false,
3162
+ "special": true
3163
+ },
3164
+ "152038": {
3165
+ "content": "|<EXTRA_TOKENS_392>|",
3166
+ "lstrip": false,
3167
+ "normalized": false,
3168
+ "rstrip": false,
3169
+ "single_word": false,
3170
+ "special": true
3171
+ },
3172
+ "152039": {
3173
+ "content": "|<EXTRA_TOKENS_393>|",
3174
+ "lstrip": false,
3175
+ "normalized": false,
3176
+ "rstrip": false,
3177
+ "single_word": false,
3178
+ "special": true
3179
+ },
3180
+ "152040": {
3181
+ "content": "|<EXTRA_TOKENS_394>|",
3182
+ "lstrip": false,
3183
+ "normalized": false,
3184
+ "rstrip": false,
3185
+ "single_word": false,
3186
+ "special": true
3187
+ },
3188
+ "152041": {
3189
+ "content": "|<EXTRA_TOKENS_395>|",
3190
+ "lstrip": false,
3191
+ "normalized": false,
3192
+ "rstrip": false,
3193
+ "single_word": false,
3194
+ "special": true
3195
+ },
3196
+ "152042": {
3197
+ "content": "|<EXTRA_TOKENS_396>|",
3198
+ "lstrip": false,
3199
+ "normalized": false,
3200
+ "rstrip": false,
3201
+ "single_word": false,
3202
+ "special": true
3203
+ },
3204
+ "152043": {
3205
+ "content": "|<EXTRA_TOKENS_397>|",
3206
+ "lstrip": false,
3207
+ "normalized": false,
3208
+ "rstrip": false,
3209
+ "single_word": false,
3210
+ "special": true
3211
+ },
3212
+ "152044": {
3213
+ "content": "|<EXTRA_TOKENS_398>|",
3214
+ "lstrip": false,
3215
+ "normalized": false,
3216
+ "rstrip": false,
3217
+ "single_word": false,
3218
+ "special": true
3219
+ },
3220
+ "152045": {
3221
+ "content": "|<EXTRA_TOKENS_399>|",
3222
+ "lstrip": false,
3223
+ "normalized": false,
3224
+ "rstrip": false,
3225
+ "single_word": false,
3226
+ "special": true
3227
+ },
3228
+ "152046": {
3229
+ "content": "|<EXTRA_TOKENS_400>|",
3230
+ "lstrip": false,
3231
+ "normalized": false,
3232
+ "rstrip": false,
3233
+ "single_word": false,
3234
+ "special": true
3235
+ },
3236
+ "152047": {
3237
+ "content": "|<EXTRA_TOKENS_401>|",
3238
+ "lstrip": false,
3239
+ "normalized": false,
3240
+ "rstrip": false,
3241
+ "single_word": false,
3242
+ "special": true
3243
+ },
3244
+ "152048": {
3245
+ "content": "|<EXTRA_TOKENS_402>|",
3246
+ "lstrip": false,
3247
+ "normalized": false,
3248
+ "rstrip": false,
3249
+ "single_word": false,
3250
+ "special": true
3251
+ },
3252
+ "152049": {
3253
+ "content": "|<EXTRA_TOKENS_403>|",
3254
+ "lstrip": false,
3255
+ "normalized": false,
3256
+ "rstrip": false,
3257
+ "single_word": false,
3258
+ "special": true
3259
+ },
3260
+ "152050": {
3261
+ "content": "|<EXTRA_TOKENS_404>|",
3262
+ "lstrip": false,
3263
+ "normalized": false,
3264
+ "rstrip": false,
3265
+ "single_word": false,
3266
+ "special": true
3267
+ },
3268
+ "152051": {
3269
+ "content": "|<EXTRA_TOKENS_405>|",
3270
+ "lstrip": false,
3271
+ "normalized": false,
3272
+ "rstrip": false,
3273
+ "single_word": false,
3274
+ "special": true
3275
+ },
3276
+ "152052": {
3277
+ "content": "|<EXTRA_TOKENS_406>|",
3278
+ "lstrip": false,
3279
+ "normalized": false,
3280
+ "rstrip": false,
3281
+ "single_word": false,
3282
+ "special": true
3283
+ },
3284
+ "152053": {
3285
+ "content": "|<EXTRA_TOKENS_407>|",
3286
+ "lstrip": false,
3287
+ "normalized": false,
3288
+ "rstrip": false,
3289
+ "single_word": false,
3290
+ "special": true
3291
+ },
3292
+ "152054": {
3293
+ "content": "|<EXTRA_TOKENS_408>|",
3294
+ "lstrip": false,
3295
+ "normalized": false,
3296
+ "rstrip": false,
3297
+ "single_word": false,
3298
+ "special": true
3299
+ },
3300
+ "152055": {
3301
+ "content": "|<EXTRA_TOKENS_409>|",
3302
+ "lstrip": false,
3303
+ "normalized": false,
3304
+ "rstrip": false,
3305
+ "single_word": false,
3306
+ "special": true
3307
+ },
3308
+ "152056": {
3309
+ "content": "|<EXTRA_TOKENS_410>|",
3310
+ "lstrip": false,
3311
+ "normalized": false,
3312
+ "rstrip": false,
3313
+ "single_word": false,
3314
+ "special": true
3315
+ },
3316
+ "152057": {
3317
+ "content": "|<EXTRA_TOKENS_411>|",
3318
+ "lstrip": false,
3319
+ "normalized": false,
3320
+ "rstrip": false,
3321
+ "single_word": false,
3322
+ "special": true
3323
+ },
3324
+ "152058": {
3325
+ "content": "|<EXTRA_TOKENS_412>|",
3326
+ "lstrip": false,
3327
+ "normalized": false,
3328
+ "rstrip": false,
3329
+ "single_word": false,
3330
+ "special": true
3331
+ },
3332
+ "152059": {
3333
+ "content": "|<EXTRA_TOKENS_413>|",
3334
+ "lstrip": false,
3335
+ "normalized": false,
3336
+ "rstrip": false,
3337
+ "single_word": false,
3338
+ "special": true
3339
+ },
3340
+ "152060": {
3341
+ "content": "|<EXTRA_TOKENS_414>|",
3342
+ "lstrip": false,
3343
+ "normalized": false,
3344
+ "rstrip": false,
3345
+ "single_word": false,
3346
+ "special": true
3347
+ },
3348
+ "152061": {
3349
+ "content": "|<EXTRA_TOKENS_415>|",
3350
+ "lstrip": false,
3351
+ "normalized": false,
3352
+ "rstrip": false,
3353
+ "single_word": false,
3354
+ "special": true
3355
+ },
3356
+ "152062": {
3357
+ "content": "|<EXTRA_TOKENS_416>|",
3358
+ "lstrip": false,
3359
+ "normalized": false,
3360
+ "rstrip": false,
3361
+ "single_word": false,
3362
+ "special": true
3363
+ },
3364
+ "152063": {
3365
+ "content": "|<EXTRA_TOKENS_417>|",
3366
+ "lstrip": false,
3367
+ "normalized": false,
3368
+ "rstrip": false,
3369
+ "single_word": false,
3370
+ "special": true
3371
+ },
3372
+ "152064": {
3373
+ "content": "<im_start>",
3374
+ "lstrip": false,
3375
+ "normalized": false,
3376
+ "rstrip": false,
3377
+ "single_word": false,
3378
+ "special": true
3379
+ },
3380
+ "152065": {
3381
+ "content": "<im_end>",
3382
+ "lstrip": false,
3383
+ "normalized": false,
3384
+ "rstrip": false,
3385
+ "single_word": false,
3386
+ "special": true
3387
+ },
3388
+ "152066": {
3389
+ "content": "<im_patch>",
3390
+ "lstrip": false,
3391
+ "normalized": false,
3392
+ "rstrip": false,
3393
+ "single_word": false,
3394
+ "special": true
3395
+ },
3396
+ "152067": {
3397
+ "content": "<im_col>",
3398
+ "lstrip": false,
3399
+ "normalized": false,
3400
+ "rstrip": false,
3401
+ "single_word": false,
3402
+ "special": true
3403
+ },
3404
+ "152068": {
3405
+ "content": "<|image|>",
3406
+ "lstrip": false,
3407
+ "normalized": false,
3408
+ "rstrip": false,
3409
+ "single_word": false,
3410
+ "special": true
3411
+ }
3412
+ },
3413
+ "additional_special_tokens": [
3414
+ "|<EXTRA_TOKENS_0>|",
3415
+ "|<EXTRA_TOKENS_1>|",
3416
+ "|<EXTRA_TOKENS_2>|",
3417
+ "|<EXTRA_TOKENS_3>|",
3418
+ "|<EXTRA_TOKENS_4>|",
3419
+ "|<EXTRA_TOKENS_5>|",
3420
+ "|<EXTRA_TOKENS_6>|",
3421
+ "|<EXTRA_TOKENS_7>|",
3422
+ "|<EXTRA_TOKENS_8>|",
3423
+ "|<EXTRA_TOKENS_9>|",
3424
+ "|<EXTRA_TOKENS_10>|",
3425
+ "|<EXTRA_TOKENS_11>|",
3426
+ "|<EXTRA_TOKENS_12>|",
3427
+ "|<EXTRA_TOKENS_13>|",
3428
+ "|<EXTRA_TOKENS_14>|",
3429
+ "|<EXTRA_TOKENS_15>|",
3430
+ "|<EXTRA_TOKENS_16>|",
3431
+ "|<EXTRA_TOKENS_17>|",
3432
+ "|<EXTRA_TOKENS_18>|",
3433
+ "|<EXTRA_TOKENS_19>|",
3434
+ "|<EXTRA_TOKENS_20>|",
3435
+ "|<EXTRA_TOKENS_21>|",
3436
+ "|<EXTRA_TOKENS_22>|",
3437
+ "|<EXTRA_TOKENS_23>|",
3438
+ "|<EXTRA_TOKENS_24>|",
3439
+ "|<EXTRA_TOKENS_25>|",
3440
+ "|<EXTRA_TOKENS_26>|",
3441
+ "|<EXTRA_TOKENS_27>|",
3442
+ "|<EXTRA_TOKENS_28>|",
3443
+ "|<EXTRA_TOKENS_29>|",
3444
+ "|<EXTRA_TOKENS_30>|",
3445
+ "|<EXTRA_TOKENS_31>|",
3446
+ "|<EXTRA_TOKENS_32>|",
3447
+ "|<EXTRA_TOKENS_33>|",
3448
+ "|<EXTRA_TOKENS_34>|",
3449
+ "|<EXTRA_TOKENS_35>|",
3450
+ "|<EXTRA_TOKENS_36>|",
3451
+ "|<EXTRA_TOKENS_37>|",
3452
+ "|<EXTRA_TOKENS_38>|",
3453
+ "|<EXTRA_TOKENS_39>|",
3454
+ "|<EXTRA_TOKENS_40>|",
3455
+ "|<EXTRA_TOKENS_41>|",
3456
+ "|<EXTRA_TOKENS_42>|",
3457
+ "|<EXTRA_TOKENS_43>|",
3458
+ "|<EXTRA_TOKENS_44>|",
3459
+ "|<EXTRA_TOKENS_45>|",
3460
+ "|<EXTRA_TOKENS_46>|",
3461
+ "|<EXTRA_TOKENS_47>|",
3462
+ "|<EXTRA_TOKENS_48>|",
3463
+ "|<EXTRA_TOKENS_49>|",
3464
+ "|<EXTRA_TOKENS_50>|",
3465
+ "|<EXTRA_TOKENS_51>|",
3466
+ "|<EXTRA_TOKENS_52>|",
3467
+ "|<EXTRA_TOKENS_53>|",
3468
+ "|<EXTRA_TOKENS_54>|",
3469
+ "|<EXTRA_TOKENS_55>|",
3470
+ "|<EXTRA_TOKENS_56>|",
3471
+ "|<EXTRA_TOKENS_57>|",
3472
+ "|<EXTRA_TOKENS_58>|",
3473
+ "|<EXTRA_TOKENS_59>|",
3474
+ "|<EXTRA_TOKENS_60>|",
3475
+ "|<EXTRA_TOKENS_61>|",
3476
+ "|<EXTRA_TOKENS_62>|",
3477
+ "|<EXTRA_TOKENS_63>|",
3478
+ "|<EXTRA_TOKENS_64>|",
3479
+ "|<EXTRA_TOKENS_65>|",
3480
+ "|<EXTRA_TOKENS_66>|",
3481
+ "|<EXTRA_TOKENS_67>|",
3482
+ "|<EXTRA_TOKENS_68>|",
3483
+ "|<EXTRA_TOKENS_69>|",
3484
+ "|<EXTRA_TOKENS_70>|",
3485
+ "|<EXTRA_TOKENS_71>|",
3486
+ "|<EXTRA_TOKENS_72>|",
3487
+ "|<EXTRA_TOKENS_73>|",
3488
+ "|<EXTRA_TOKENS_74>|",
3489
+ "|<EXTRA_TOKENS_75>|",
3490
+ "|<EXTRA_TOKENS_76>|",
3491
+ "|<EXTRA_TOKENS_77>|",
3492
+ "|<EXTRA_TOKENS_78>|",
3493
+ "|<EXTRA_TOKENS_79>|",
3494
+ "|<EXTRA_TOKENS_80>|",
3495
+ "|<EXTRA_TOKENS_81>|",
3496
+ "|<EXTRA_TOKENS_82>|",
3497
+ "|<EXTRA_TOKENS_83>|",
3498
+ "|<EXTRA_TOKENS_84>|",
3499
+ "|<EXTRA_TOKENS_85>|",
3500
+ "|<EXTRA_TOKENS_86>|",
3501
+ "|<EXTRA_TOKENS_87>|",
3502
+ "|<EXTRA_TOKENS_88>|",
3503
+ "|<EXTRA_TOKENS_89>|",
3504
+ "|<EXTRA_TOKENS_90>|",
3505
+ "|<EXTRA_TOKENS_91>|",
3506
+ "|<EXTRA_TOKENS_92>|",
3507
+ "|<EXTRA_TOKENS_93>|",
3508
+ "|<EXTRA_TOKENS_94>|",
3509
+ "|<EXTRA_TOKENS_95>|",
3510
+ "|<EXTRA_TOKENS_96>|",
3511
+ "|<EXTRA_TOKENS_97>|",
3512
+ "|<EXTRA_TOKENS_98>|",
3513
+ "|<EXTRA_TOKENS_99>|",
3514
+ "|<EXTRA_TOKENS_100>|",
3515
+ "|<EXTRA_TOKENS_101>|",
3516
+ "|<EXTRA_TOKENS_102>|",
3517
+ "|<EXTRA_TOKENS_103>|",
3518
+ "|<EXTRA_TOKENS_104>|",
3519
+ "|<EXTRA_TOKENS_105>|",
3520
+ "|<EXTRA_TOKENS_106>|",
3521
+ "|<EXTRA_TOKENS_107>|",
3522
+ "|<EXTRA_TOKENS_108>|",
3523
+ "|<EXTRA_TOKENS_109>|",
3524
+ "|<EXTRA_TOKENS_110>|",
3525
+ "|<EXTRA_TOKENS_111>|",
3526
+ "|<EXTRA_TOKENS_112>|",
3527
+ "|<EXTRA_TOKENS_113>|",
3528
+ "|<EXTRA_TOKENS_114>|",
3529
+ "|<EXTRA_TOKENS_115>|",
3530
+ "|<EXTRA_TOKENS_116>|",
3531
+ "|<EXTRA_TOKENS_117>|",
3532
+ "|<EXTRA_TOKENS_118>|",
3533
+ "|<EXTRA_TOKENS_119>|",
3534
+ "|<EXTRA_TOKENS_120>|",
3535
+ "|<EXTRA_TOKENS_121>|",
3536
+ "|<EXTRA_TOKENS_122>|",
3537
+ "|<EXTRA_TOKENS_123>|",
3538
+ "|<EXTRA_TOKENS_124>|",
3539
+ "|<EXTRA_TOKENS_125>|",
3540
+ "|<EXTRA_TOKENS_126>|",
3541
+ "|<EXTRA_TOKENS_127>|",
3542
+ "|<EXTRA_TOKENS_128>|",
3543
+ "|<EXTRA_TOKENS_129>|",
3544
+ "|<EXTRA_TOKENS_130>|",
3545
+ "|<EXTRA_TOKENS_131>|",
3546
+ "|<EXTRA_TOKENS_132>|",
3547
+ "|<EXTRA_TOKENS_133>|",
3548
+ "|<EXTRA_TOKENS_134>|",
3549
+ "|<EXTRA_TOKENS_135>|",
3550
+ "|<EXTRA_TOKENS_136>|",
3551
+ "|<EXTRA_TOKENS_137>|",
3552
+ "|<EXTRA_TOKENS_138>|",
3553
+ "|<EXTRA_TOKENS_139>|",
3554
+ "|<EXTRA_TOKENS_140>|",
3555
+ "|<EXTRA_TOKENS_141>|",
3556
+ "|<EXTRA_TOKENS_142>|",
3557
+ "|<EXTRA_TOKENS_143>|",
3558
+ "|<EXTRA_TOKENS_144>|",
3559
+ "|<EXTRA_TOKENS_145>|",
3560
+ "|<EXTRA_TOKENS_146>|",
3561
+ "|<EXTRA_TOKENS_147>|",
3562
+ "|<EXTRA_TOKENS_148>|",
3563
+ "|<EXTRA_TOKENS_149>|",
3564
+ "|<EXTRA_TOKENS_150>|",
3565
+ "|<EXTRA_TOKENS_151>|",
3566
+ "|<EXTRA_TOKENS_152>|",
3567
+ "|<EXTRA_TOKENS_153>|",
3568
+ "|<EXTRA_TOKENS_154>|",
3569
+ "|<EXTRA_TOKENS_155>|",
3570
+ "|<EXTRA_TOKENS_156>|",
3571
+ "|<EXTRA_TOKENS_157>|",
3572
+ "|<EXTRA_TOKENS_158>|",
3573
+ "|<EXTRA_TOKENS_159>|",
3574
+ "|<EXTRA_TOKENS_160>|",
3575
+ "|<EXTRA_TOKENS_161>|",
3576
+ "|<EXTRA_TOKENS_162>|",
3577
+ "|<EXTRA_TOKENS_163>|",
3578
+ "|<EXTRA_TOKENS_164>|",
3579
+ "|<EXTRA_TOKENS_165>|",
3580
+ "|<EXTRA_TOKENS_166>|",
3581
+ "|<EXTRA_TOKENS_167>|",
3582
+ "|<EXTRA_TOKENS_168>|",
3583
+ "|<EXTRA_TOKENS_169>|",
3584
+ "|<EXTRA_TOKENS_170>|",
3585
+ "|<EXTRA_TOKENS_171>|",
3586
+ "|<EXTRA_TOKENS_172>|",
3587
+ "|<EXTRA_TOKENS_173>|",
3588
+ "|<EXTRA_TOKENS_174>|",
3589
+ "|<EXTRA_TOKENS_175>|",
3590
+ "|<EXTRA_TOKENS_176>|",
3591
+ "|<EXTRA_TOKENS_177>|",
3592
+ "|<EXTRA_TOKENS_178>|",
3593
+ "|<EXTRA_TOKENS_179>|",
3594
+ "|<EXTRA_TOKENS_180>|",
3595
+ "|<EXTRA_TOKENS_181>|",
3596
+ "|<EXTRA_TOKENS_182>|",
3597
+ "|<EXTRA_TOKENS_183>|",
3598
+ "|<EXTRA_TOKENS_184>|",
3599
+ "|<EXTRA_TOKENS_185>|",
3600
+ "|<EXTRA_TOKENS_186>|",
3601
+ "|<EXTRA_TOKENS_187>|",
3602
+ "|<EXTRA_TOKENS_188>|",
3603
+ "|<EXTRA_TOKENS_189>|",
3604
+ "|<EXTRA_TOKENS_190>|",
3605
+ "|<EXTRA_TOKENS_191>|",
3606
+ "|<EXTRA_TOKENS_192>|",
3607
+ "|<EXTRA_TOKENS_193>|",
3608
+ "|<EXTRA_TOKENS_194>|",
3609
+ "|<EXTRA_TOKENS_195>|",
3610
+ "|<EXTRA_TOKENS_196>|",
3611
+ "|<EXTRA_TOKENS_197>|",
3612
+ "|<EXTRA_TOKENS_198>|",
3613
+ "|<EXTRA_TOKENS_199>|",
3614
+ "|<EXTRA_TOKENS_200>|",
3615
+ "|<EXTRA_TOKENS_201>|",
3616
+ "|<EXTRA_TOKENS_202>|",
3617
+ "|<EXTRA_TOKENS_203>|",
3618
+ "|<EXTRA_TOKENS_204>|",
3619
+ "|<EXTRA_TOKENS_205>|",
3620
+ "|<EXTRA_TOKENS_206>|",
3621
+ "|<EXTRA_TOKENS_207>|",
3622
+ "|<EXTRA_TOKENS_208>|",
3623
+ "|<EXTRA_TOKENS_209>|",
3624
+ "|<EXTRA_TOKENS_210>|",
3625
+ "|<EXTRA_TOKENS_211>|",
3626
+ "|<EXTRA_TOKENS_212>|",
3627
+ "|<EXTRA_TOKENS_213>|",
3628
+ "|<EXTRA_TOKENS_214>|",
3629
+ "|<EXTRA_TOKENS_215>|",
3630
+ "|<EXTRA_TOKENS_216>|",
3631
+ "|<EXTRA_TOKENS_217>|",
3632
+ "|<EXTRA_TOKENS_218>|",
3633
+ "|<EXTRA_TOKENS_219>|",
3634
+ "|<EXTRA_TOKENS_220>|",
3635
+ "|<EXTRA_TOKENS_221>|",
3636
+ "|<EXTRA_TOKENS_222>|",
3637
+ "|<EXTRA_TOKENS_223>|",
3638
+ "|<EXTRA_TOKENS_224>|",
3639
+ "|<EXTRA_TOKENS_225>|",
3640
+ "|<EXTRA_TOKENS_226>|",
3641
+ "|<EXTRA_TOKENS_227>|",
3642
+ "|<EXTRA_TOKENS_228>|",
3643
+ "|<EXTRA_TOKENS_229>|",
3644
+ "|<EXTRA_TOKENS_230>|",
3645
+ "|<EXTRA_TOKENS_231>|",
3646
+ "|<EXTRA_TOKENS_232>|",
3647
+ "|<EXTRA_TOKENS_233>|",
3648
+ "|<EXTRA_TOKENS_234>|",
3649
+ "|<EXTRA_TOKENS_235>|",
3650
+ "|<EXTRA_TOKENS_236>|",
3651
+ "|<EXTRA_TOKENS_237>|",
3652
+ "|<EXTRA_TOKENS_238>|",
3653
+ "|<EXTRA_TOKENS_239>|",
3654
+ "|<EXTRA_TOKENS_240>|",
3655
+ "|<EXTRA_TOKENS_241>|",
3656
+ "|<EXTRA_TOKENS_242>|",
3657
+ "|<EXTRA_TOKENS_243>|",
3658
+ "|<EXTRA_TOKENS_244>|",
3659
+ "|<EXTRA_TOKENS_245>|",
3660
+ "|<EXTRA_TOKENS_246>|",
3661
+ "|<EXTRA_TOKENS_247>|",
3662
+ "|<EXTRA_TOKENS_248>|",
3663
+ "|<EXTRA_TOKENS_249>|",
3664
+ "|<EXTRA_TOKENS_250>|",
3665
+ "|<EXTRA_TOKENS_251>|",
3666
+ "|<EXTRA_TOKENS_252>|",
3667
+ "|<EXTRA_TOKENS_253>|",
3668
+ "|<EXTRA_TOKENS_254>|",
3669
+ "|<EXTRA_TOKENS_255>|",
3670
+ "|<EXTRA_TOKENS_256>|",
3671
+ "|<EXTRA_TOKENS_257>|",
3672
+ "|<EXTRA_TOKENS_258>|",
3673
+ "|<EXTRA_TOKENS_259>|",
3674
+ "|<EXTRA_TOKENS_260>|",
3675
+ "|<EXTRA_TOKENS_261>|",
3676
+ "|<EXTRA_TOKENS_262>|",
3677
+ "|<EXTRA_TOKENS_263>|",
3678
+ "|<EXTRA_TOKENS_264>|",
3679
+ "|<EXTRA_TOKENS_265>|",
3680
+ "|<EXTRA_TOKENS_266>|",
3681
+ "|<EXTRA_TOKENS_267>|",
3682
+ "|<EXTRA_TOKENS_268>|",
3683
+ "|<EXTRA_TOKENS_269>|",
3684
+ "|<EXTRA_TOKENS_270>|",
3685
+ "|<EXTRA_TOKENS_271>|",
3686
+ "|<EXTRA_TOKENS_272>|",
3687
+ "|<EXTRA_TOKENS_273>|",
3688
+ "|<EXTRA_TOKENS_274>|",
3689
+ "|<EXTRA_TOKENS_275>|",
3690
+ "|<EXTRA_TOKENS_276>|",
3691
+ "|<EXTRA_TOKENS_277>|",
3692
+ "|<EXTRA_TOKENS_278>|",
3693
+ "|<EXTRA_TOKENS_279>|",
3694
+ "|<EXTRA_TOKENS_280>|",
3695
+ "|<EXTRA_TOKENS_281>|",
3696
+ "|<EXTRA_TOKENS_282>|",
3697
+ "|<EXTRA_TOKENS_283>|",
3698
+ "|<EXTRA_TOKENS_284>|",
3699
+ "|<EXTRA_TOKENS_285>|",
3700
+ "|<EXTRA_TOKENS_286>|",
3701
+ "|<EXTRA_TOKENS_287>|",
3702
+ "|<EXTRA_TOKENS_288>|",
3703
+ "|<EXTRA_TOKENS_289>|",
3704
+ "|<EXTRA_TOKENS_290>|",
3705
+ "|<EXTRA_TOKENS_291>|",
3706
+ "|<EXTRA_TOKENS_292>|",
3707
+ "|<EXTRA_TOKENS_293>|",
3708
+ "|<EXTRA_TOKENS_294>|",
3709
+ "|<EXTRA_TOKENS_295>|",
3710
+ "|<EXTRA_TOKENS_296>|",
3711
+ "|<EXTRA_TOKENS_297>|",
3712
+ "|<EXTRA_TOKENS_298>|",
3713
+ "|<EXTRA_TOKENS_299>|",
3714
+ "|<EXTRA_TOKENS_300>|",
3715
+ "|<EXTRA_TOKENS_301>|",
3716
+ "|<EXTRA_TOKENS_302>|",
3717
+ "|<EXTRA_TOKENS_303>|",
3718
+ "|<EXTRA_TOKENS_304>|",
3719
+ "|<EXTRA_TOKENS_305>|",
3720
+ "|<EXTRA_TOKENS_306>|",
3721
+ "|<EXTRA_TOKENS_307>|",
3722
+ "|<EXTRA_TOKENS_308>|",
3723
+ "|<EXTRA_TOKENS_309>|",
3724
+ "|<EXTRA_TOKENS_310>|",
3725
+ "|<EXTRA_TOKENS_311>|",
3726
+ "|<EXTRA_TOKENS_312>|",
3727
+ "|<EXTRA_TOKENS_313>|",
3728
+ "|<EXTRA_TOKENS_314>|",
3729
+ "|<EXTRA_TOKENS_315>|",
3730
+ "|<EXTRA_TOKENS_316>|",
3731
+ "|<EXTRA_TOKENS_317>|",
3732
+ "|<EXTRA_TOKENS_318>|",
3733
+ "|<EXTRA_TOKENS_319>|",
3734
+ "|<EXTRA_TOKENS_320>|",
3735
+ "|<EXTRA_TOKENS_321>|",
3736
+ "|<EXTRA_TOKENS_322>|",
3737
+ "|<EXTRA_TOKENS_323>|",
3738
+ "|<EXTRA_TOKENS_324>|",
3739
+ "|<EXTRA_TOKENS_325>|",
3740
+ "|<EXTRA_TOKENS_326>|",
3741
+ "|<EXTRA_TOKENS_327>|",
3742
+ "|<EXTRA_TOKENS_328>|",
3743
+ "|<EXTRA_TOKENS_329>|",
3744
+ "|<EXTRA_TOKENS_330>|",
3745
+ "|<EXTRA_TOKENS_331>|",
3746
+ "|<EXTRA_TOKENS_332>|",
3747
+ "|<EXTRA_TOKENS_333>|",
3748
+ "|<EXTRA_TOKENS_334>|",
3749
+ "|<EXTRA_TOKENS_335>|",
3750
+ "|<EXTRA_TOKENS_336>|",
3751
+ "|<EXTRA_TOKENS_337>|",
3752
+ "|<EXTRA_TOKENS_338>|",
3753
+ "|<EXTRA_TOKENS_339>|",
3754
+ "|<EXTRA_TOKENS_340>|",
3755
+ "|<EXTRA_TOKENS_341>|",
3756
+ "|<EXTRA_TOKENS_342>|",
3757
+ "|<EXTRA_TOKENS_343>|",
3758
+ "|<EXTRA_TOKENS_344>|",
3759
+ "|<EXTRA_TOKENS_345>|",
3760
+ "|<EXTRA_TOKENS_346>|",
3761
+ "|<EXTRA_TOKENS_347>|",
3762
+ "|<EXTRA_TOKENS_348>|",
3763
+ "|<EXTRA_TOKENS_349>|",
3764
+ "|<EXTRA_TOKENS_350>|",
3765
+ "|<EXTRA_TOKENS_351>|",
3766
+ "|<EXTRA_TOKENS_352>|",
3767
+ "|<EXTRA_TOKENS_353>|",
3768
+ "|<EXTRA_TOKENS_354>|",
3769
+ "|<EXTRA_TOKENS_355>|",
3770
+ "|<EXTRA_TOKENS_356>|",
3771
+ "|<EXTRA_TOKENS_357>|",
3772
+ "|<EXTRA_TOKENS_358>|",
3773
+ "|<EXTRA_TOKENS_359>|",
3774
+ "|<EXTRA_TOKENS_360>|",
3775
+ "|<EXTRA_TOKENS_361>|",
3776
+ "|<EXTRA_TOKENS_362>|",
3777
+ "|<EXTRA_TOKENS_363>|",
3778
+ "|<EXTRA_TOKENS_364>|",
3779
+ "|<EXTRA_TOKENS_365>|",
3780
+ "|<EXTRA_TOKENS_366>|",
3781
+ "|<EXTRA_TOKENS_367>|",
3782
+ "|<EXTRA_TOKENS_368>|",
3783
+ "|<EXTRA_TOKENS_369>|",
3784
+ "|<EXTRA_TOKENS_370>|",
3785
+ "|<EXTRA_TOKENS_371>|",
3786
+ "|<EXTRA_TOKENS_372>|",
3787
+ "|<EXTRA_TOKENS_373>|",
3788
+ "|<EXTRA_TOKENS_374>|",
3789
+ "|<EXTRA_TOKENS_375>|",
3790
+ "|<EXTRA_TOKENS_376>|",
3791
+ "|<EXTRA_TOKENS_377>|",
3792
+ "|<EXTRA_TOKENS_378>|",
3793
+ "|<EXTRA_TOKENS_379>|",
3794
+ "|<EXTRA_TOKENS_380>|",
3795
+ "|<EXTRA_TOKENS_381>|",
3796
+ "|<EXTRA_TOKENS_382>|",
3797
+ "|<EXTRA_TOKENS_383>|",
3798
+ "|<EXTRA_TOKENS_384>|",
3799
+ "|<EXTRA_TOKENS_385>|",
3800
+ "|<EXTRA_TOKENS_386>|",
3801
+ "|<EXTRA_TOKENS_387>|",
3802
+ "|<EXTRA_TOKENS_388>|",
3803
+ "|<EXTRA_TOKENS_389>|",
3804
+ "|<EXTRA_TOKENS_390>|",
3805
+ "|<EXTRA_TOKENS_391>|",
3806
+ "|<EXTRA_TOKENS_392>|",
3807
+ "|<EXTRA_TOKENS_393>|",
3808
+ "|<EXTRA_TOKENS_394>|",
3809
+ "|<EXTRA_TOKENS_395>|",
3810
+ "|<EXTRA_TOKENS_396>|",
3811
+ "|<EXTRA_TOKENS_397>|",
3812
+ "|<EXTRA_TOKENS_398>|",
3813
+ "|<EXTRA_TOKENS_399>|",
3814
+ "|<EXTRA_TOKENS_400>|",
3815
+ "|<EXTRA_TOKENS_401>|",
3816
+ "|<EXTRA_TOKENS_402>|",
3817
+ "|<EXTRA_TOKENS_403>|",
3818
+ "|<EXTRA_TOKENS_404>|",
3819
+ "|<EXTRA_TOKENS_405>|",
3820
+ "|<EXTRA_TOKENS_406>|",
3821
+ "|<EXTRA_TOKENS_407>|",
3822
+ "|<EXTRA_TOKENS_408>|",
3823
+ "|<EXTRA_TOKENS_409>|",
3824
+ "|<EXTRA_TOKENS_410>|",
3825
+ "|<EXTRA_TOKENS_411>|",
3826
+ "|<EXTRA_TOKENS_412>|",
3827
+ "|<EXTRA_TOKENS_413>|",
3828
+ "|<EXTRA_TOKENS_414>|",
3829
+ "|<EXTRA_TOKENS_415>|",
3830
+ "|<EXTRA_TOKENS_416>|",
3831
+ "|<EXTRA_TOKENS_417>|",
3832
+ "<im_start>",
3833
+ "<im_end>",
3834
+ "<im_patch>",
3835
+ "<im_col>",
3836
+ "<|image|>"
3837
+ ],
3838
+ "auto_map": {
3839
+ "AutoProcessor": "preprocessing_molmo.MolmoProcessor"
3840
+ },
3841
+ "bos_token": null,
3842
+ "chat_template": "{% for message in messages -%}\n {%- if (loop.index % 2 == 1 and message['role'] != 'user') or \n (loop.index % 2 == 0 and message['role'].lower() != 'assistant') -%}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {%- endif -%}\n {{ message['role'].capitalize() + ': ' + message['content'] }}\n {%- if not loop.last -%}\n {{ ' ' }}\n {%- endif %}\n {%- endfor -%}\n {%- if add_generation_prompt -%}\n {{ ' Assistant:' }}\n {%- endif %}",
3843
+ "clean_up_tokenization_spaces": false,
3844
+ "eos_token": "<|endoftext|>",
3845
+ "errors": "replace",
3846
+ "extra_special_tokens": {},
3847
+ "model_max_length": 32768,
3848
+ "pad_token": "<|endoftext|>",
3849
+ "processor_class": "MolmoProcessor",
3850
+ "split_special_tokens": false,
3851
+ "tokenizer_class": "Qwen2Tokenizer",
3852
+ "unk_token": null
3853
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff