 
			
		Upload tokenizer
		87ea6e6
		verified
		
				
					
						
						|  | { | 
					
						
						|  | "bos_token": { | 
					
						
						|  | "content": "<bos>", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "eos_token": { | 
					
						
						|  | "content": "<eos>", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "pad_token": { | 
					
						
						|  | "content": "<pad>", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | }, | 
					
						
						|  | "unk_token": { | 
					
						
						|  | "content": "<unk>", | 
					
						
						|  | "lstrip": false, | 
					
						
						|  | "normalized": false, | 
					
						
						|  | "rstrip": false, | 
					
						
						|  | "single_word": false | 
					
						
						|  | } | 
					
						
						|  | } | 
					
						
						|  |  |