fm4bio-ning commited on
Commit
6c245c7
·
verified ·
1 Parent(s): b6cb3ec

Upload tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +8 -2
tokenizer_config.json CHANGED
@@ -43,5 +43,11 @@
43
  "sep_token": "[SEP]",
44
  "tokenizer_class": "FM4BioTokenizer",
45
  "unk_token": "-",
46
- "version": "v1"
47
- }
 
 
 
 
 
 
 
43
  "sep_token": "[SEP]",
44
  "tokenizer_class": "FM4BioTokenizer",
45
  "unk_token": "-",
46
+ "version": "v1",
47
+ "auto_map": {
48
+ "AutoTokenizer": [
49
+ "tokenization_fm4bio.FM4BioTokenizer",
50
+ null
51
+ ]
52
+ }
53
+ }