


'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__    	=     logging.get_logger(__name__)

lowerCamelCase__    	=     '▁'

lowerCamelCase__    	=     {'vocab_file': 'sentencepiece.bpe.model'}

lowerCamelCase__    	=     {
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

lowerCamelCase__    	=     {
    'facebook/xglm-564M': 2_048,
}






class 	lowerCAmelCase__			(			UpperCAmelCase__	):
			lowerCAmelCase :	str          =     VOCAB_FILES_NAMES
			lowerCAmelCase :	Dict          =     PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase :	Any          =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase :	Optional[int]          =     ["input_ids", "attention_mask"]




			def __init__(		self  :     List[str]	,  lowerCamelCase__  :     List[str]	,  lowerCamelCase__  :     List[Any]="<s>"	,  lowerCamelCase__  :     Tuple="</s>"	,  lowerCamelCase__  :     List[Any]="</s>"	,  lowerCamelCase__  :     List[Any]="<s>"	,  lowerCamelCase__  :     Union[str, Any]="<unk>"	,  lowerCamelCase__  :     List[str]="<pad>"	,  lowerCamelCase__  :     Optional[Dict[str, Any]] = None	,  **lowerCamelCase__  :     Optional[int]	,  )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[Any]								=      {} if sp_model_kwargs is None else sp_model_kwargs

					# Compatibility with the original tokenizer
					_UpperCAmelCase		:       List[Any]								=      7
					_UpperCAmelCase		:       Dict								=      [F"""<madeupword{i}>""" for i in range(self.num_madeup_words )]

					_UpperCAmelCase		:       Optional[int]								=      kwargs.get("additional_special_tokens"	,  [] )
					kwargs["additional_special_tokens"] += [
					    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
					]

					super().__init__(
					    bos_token=lowerCamelCase__	,  eos_token=lowerCamelCase__	,  unk_token=lowerCamelCase__	,  sep_token=lowerCamelCase__	,  cls_token=lowerCamelCase__	,  pad_token=lowerCamelCase__	,  sp_model_kwargs=self.sp_model_kwargs	,  **lowerCamelCase__	,  )

					_UpperCAmelCase		:       Dict								=      spm.SentencePieceProcessor(**self.sp_model_kwargs )
					self.sp_model.Load(str(lowerCamelCase__ ) )
					_UpperCAmelCase		:       Optional[int]								=      vocab_file

					# Original fairseq vocab and spm vocab must be "aligned":
					# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
					# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
					# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
					# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

					# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
					_UpperCAmelCase		:       str								=      1

					# Mimic fairseq token-to-id alignment for the first 4 token
					_UpperCAmelCase		:       Dict								=      {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

					_UpperCAmelCase		:       List[str]								=      len(self.sp_model )
					_UpperCAmelCase		:       List[str]								=      {F"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words )}
					self.fairseq_tokens_to_ids.update(lowerCamelCase__ )

					_UpperCAmelCase		:       str								=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}




			def __getstate__(		self  :     int )   ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      self.__dict__.copy()
					_UpperCAmelCase		:       List[str]								=      None
					_UpperCAmelCase		:       str								=      self.sp_model.serialized_model_proto()
					return state




			def __setstate__(		self  :     Dict	,  lowerCamelCase__  :     Optional[int] )   ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase		:       List[Any]								=      d

					# for backward compatibility
					if not hasattr(self	,  "sp_model_kwargs" ):
							_UpperCAmelCase		:       Optional[int]								=      {}

					_UpperCAmelCase		:       Optional[int]								=      spm.SentencePieceProcessor(**self.sp_model_kwargs )
					self.sp_model.LoadFromSerializedProto(self.sp_model_proto )




			def 							lowerCAmelCase__    (		self  :     Any	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					if token_ids_a is None:
							return [self.sep_token_id] + token_ids_a
					_UpperCAmelCase		:       Dict								=      [self.sep_token_id]
					return sep + token_ids_a + sep + sep + token_ids_a




			def 							lowerCAmelCase__    (		self  :     List[Any]	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None	,  lowerCamelCase__  :     bool = False )   ->List[int]:



					'''simple docstring'''







					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCamelCase__	,  token_ids_a=lowerCamelCase__	,  already_has_special_tokens=lowerCamelCase__ )

					if token_ids_a is None:
							return [1] + ([0] * len(lowerCamelCase__ ))
					return [1] + ([0] * len(lowerCamelCase__ )) + [1, 1] + ([0] * len(lowerCamelCase__ ))




			def 							lowerCAmelCase__    (		self  :     Optional[Any]	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       str								=      [self.sep_token_id]

					if token_ids_a is None:
							return len(sep + token_ids_a ) * [0]
					return len(sep + token_ids_a + sep + sep + token_ids_a ) * [0]




			@property
			def 							lowerCAmelCase__    (		self  :     List[Any] )   ->Optional[Any]:



					'''simple docstring'''







					return len(self.sp_model ) + self.fairseq_offset + self.num_madeup_words




			def 							lowerCAmelCase__    (		self  :     Union[str, Any] )   ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      {self.convert_ids_to_tokens(lowerCamelCase__ ): i for i in range(self.vocab_size )}
					vocab.update(self.added_tokens_encoder )
					return vocab




			def 							lowerCAmelCase__    (		self  :     int	,  lowerCamelCase__  :     str )   ->List[str]:



					'''simple docstring'''







					return self.sp_model.encode(lowerCamelCase__	,  out_type=lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     Optional[int]	,  lowerCamelCase__  :     str )   ->List[str]:



					'''simple docstring'''







					if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
					_UpperCAmelCase		:       int								=      self.sp_model.PieceToId(lowerCamelCase__ )

					# Need to return unknown token if the SP model returned 0
					return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




			def 							lowerCAmelCase__    (		self  :     Dict	,  lowerCamelCase__  :     Dict )   ->Optional[Any]:



					'''simple docstring'''







					if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(index - self.fairseq_offset )




			def 							lowerCAmelCase__    (		self  :     Optional[Any]	,  lowerCamelCase__  :     Tuple )   ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[Any]								=      "".join(lowerCamelCase__ ).replace(lowerCamelCase__	,  " " ).strip()
					return out_string






			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Optional[str] = None )   ->Tuple[str]:



					'''simple docstring'''







					if not os.path.isdir(lowerCamelCase__ ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
							return
					_UpperCAmelCase		:       Optional[Any]								=      os.path.join(
					    lowerCamelCase__	,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCamelCase__ ) and os.path.isfile(self.vocab_file ):
							copyfile(self.vocab_file	,  lowerCamelCase__ )
					elif not os.path.isfile(self.vocab_file ):
							with open(lowerCamelCase__	,  "wb" ) as fi:
									_UpperCAmelCase		:       Union[str, Any]								=      self.sp_model.serialized_model_proto()
									fi.write(lowerCamelCase__ )

					return (out_vocab_file,)







'''simple docstring'''


import os






def  __lowerCAmelCase ():
		_UpperCAmelCase		:       List[Any]								=      os.path.join(os.path.dirname(__lowerCAmelCase   )      , "num.txt"   )
		with open(__lowerCAmelCase   ) as file_hand:
				return str(sum(int(__lowerCAmelCase   ) for line in file_hand   )   )[:10]


if __name__ == "__main__":
						print(solution())



