




import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase	:						str			=     logging.get_logger(__name__)

_UpperCAmelCase	:						Optional[int]			=     "▁"

_UpperCAmelCase	:						Dict			=     {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_UpperCAmelCase	:						Optional[int]			=     {
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_UpperCAmelCase	:						int			=     {
    "facebook/m2m100_418M": 1024,
}

# fmt: off
_UpperCAmelCase	:						Union[str, Any]			=     {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						VOCAB_FILES_NAMES
       _a 					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a 					=						PRETRAINED_VOCAB_FILES_MAP
       _a 					=						['''input_ids''', '''attention_mask''']

       _a 					=						[]
       _a 					=						[]






       def __init__(   self:					List[str]       ,					_lowerCAmelCase:					List[Any]       ,					_lowerCAmelCase:					Union[str, Any]       ,					_lowerCAmelCase:					Optional[Any]=None       ,					_lowerCAmelCase:					Tuple=None       ,					_lowerCAmelCase:					List[str]="<s>"       ,					_lowerCAmelCase:					Optional[int]="</s>"       ,					_lowerCAmelCase:					Any="</s>"       ,					_lowerCAmelCase:					str="<pad>"       ,					_lowerCAmelCase:					int="<unk>"       ,					_lowerCAmelCase:					Optional[int]="m2m100"       ,					_lowerCAmelCase:					Optional[Dict[str, Any]] = None       ,					_lowerCAmelCase:					Optional[Any]=8       ,					**_lowerCAmelCase:					Optional[Any]       ,					):
         lowercase    :Optional[int]      	=						{} if sp_model_kwargs is None else sp_model_kwargs

         lowercase    :str      	=						language_codes
         lowercase    :Any      	=						FAIRSEQ_LANGUAGE_CODES[language_codes]
         lowercase    :Tuple      	=						{lang_code: F"__{lang_code}__" for lang_code in fairseq_language_code}

         lowercase    :Dict      	=						kwargs.get("additional_special_tokens"       ,					[]				)
         kwargs["additional_special_tokens"] += [
             self.get_lang_token(_lowerCAmelCase				)
             for lang_code in fairseq_language_code
             if self.get_lang_token(_lowerCAmelCase				) not in kwargs["additional_special_tokens"]
         ]

         super().__init__(
             src_lang=_lowerCAmelCase       ,					tgt_lang=_lowerCAmelCase       ,					bos_token=_lowerCAmelCase       ,					eos_token=_lowerCAmelCase       ,					sep_token=_lowerCAmelCase       ,					unk_token=_lowerCAmelCase       ,					pad_token=_lowerCAmelCase       ,					language_codes=_lowerCAmelCase       ,					sp_model_kwargs=self.sp_model_kwargs       ,					num_madeup_words=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)

         lowercase    :Tuple      	=						vocab_file
         lowercase    :List[Any]      	=						load_json(_lowerCAmelCase				)
         lowercase    :Dict      	=						{v: k for k, v in self.encoder.items()}
         lowercase    :Any      	=						spm_file
         lowercase    :Any      	=						load_spm(_lowerCAmelCase       ,					self.sp_model_kwargs				)

         lowercase    :str      	=						len(self.encoder				)

         lowercase    :List[Any]      	=						{
             self.get_lang_token(_lowerCAmelCase				): self.encoder_size + i for i, lang_code in enumerate(_lowerCAmelCase				)
         }
         lowercase    :List[str]      	=						{lang_code: self.encoder_size + i for i, lang_code in enumerate(_lowerCAmelCase				)}
         lowercase    :Any      	=						{v: k for k, v in self.lang_token_to_id.items()}

         lowercase    :List[str]      	=						src_lang if src_lang is not None else "en"
         lowercase    :List[Any]      	=						tgt_lang
         lowercase    :Optional[Any]      	=						self.get_lang_id(self._src_lang				)
         self.set_src_lang_special_tokens(self._src_lang				)

         lowercase    :Union[str, Any]      	=						num_madeup_words






       @property
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         return len(self.encoder				) + len(self.lang_token_to_id				)






       @property
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         return self._src_lang






       @src_lang.setter
       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]       ,					_lowerCAmelCase:					str				):
         lowercase    :List[Any]      	=						new_src_lang
         self.set_src_lang_special_tokens(self._src_lang				)






       def 					SCREAMING_SNAKE_CASE    (   self:					str       ,					_lowerCAmelCase:					str				):
         return self.sp_model.encode(_lowerCAmelCase       ,					out_type=_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]       ,					_lowerCAmelCase:					Dict				):
         if token in self.lang_token_to_id:
           return self.lang_token_to_id[token]
         return self.encoder.get(_lowerCAmelCase       ,					self.encoder[self.unk_token]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]       ,					_lowerCAmelCase:					int				):
         if index in self.id_to_lang_token:
           return self.id_to_lang_token[index]
         return self.decoder.get(_lowerCAmelCase       ,					self.unk_token				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict       ,					_lowerCAmelCase:					Optional[int]				):
         lowercase    :Tuple      	=						[]
         lowercase    :Optional[Any]      	=						""
         for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
             out_string += self.sp_model.decode(_lowerCAmelCase				) + token
             lowercase    :Dict      	=						[]
           else:
             current_sub_tokens.append(_lowerCAmelCase				)
         out_string += self.sp_model.decode(_lowerCAmelCase				)
         return out_string.strip()






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]       ,					_lowerCAmelCase:					List[int]       ,					_lowerCAmelCase:					Optional[List[int]] = None       ,					_lowerCAmelCase:					bool = False				):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_lowerCAmelCase       ,					token_ids_a=_lowerCAmelCase       ,					already_has_special_tokens=_lowerCAmelCase				)

         lowercase    :Optional[int]      	=						[1] * len(self.prefix_tokens				)
         lowercase    :Optional[int]      	=						[1] * len(self.suffix_tokens				)
         if token_ids_a is None:
           return prefix_ones + ([0] * len(_lowerCAmelCase				)) + suffix_ones
         return prefix_ones + ([0] * len(_lowerCAmelCase				)) + ([0] * len(_lowerCAmelCase				)) + suffix_ones






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict       ,					_lowerCAmelCase:					List[int]       ,					_lowerCAmelCase:					Optional[List[int]] = None				):
         if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         lowercase    :Tuple      	=						{self.convert_ids_to_tokens(_lowerCAmelCase				): i for i in range(self.vocab_size				)}
         vocab.update(self.added_tokens_encoder				)
         return vocab






       def __getstate__(   self:					Any				):
         lowercase    :Optional[int]      	=						self.__dict__.copy()
         lowercase    :Optional[Any]      	=						None
         return state






       def __setstate__(   self:					Union[str, Any]       ,					_lowerCAmelCase:					Dict				):
         lowercase    :Optional[Any]      	=						d

         # for backward compatibility
         if not hasattr(self       ,					"sp_model_kwargs"				):
           lowercase    :str      	=						{}

         lowercase    :str      	=						load_spm(self.spm_file       ,					self.sp_model_kwargs				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					str       ,					_lowerCAmelCase:					Optional[str] = None				):
         lowercase    :Tuple      	=						Path(_lowerCAmelCase				)
         if not save_dir.is_dir():
           raise OSError(F"{save_directory} should be a directory"				)
         lowercase    :List[Any]      	=						save_dir / (
             (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
         )
         lowercase    :Any      	=						save_dir / (
             (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
         )

         save_json(self.encoder       ,					_lowerCAmelCase				)

         if os.path.abspath(self.spm_file				) != os.path.abspath(_lowerCAmelCase				) and os.path.isfile(self.spm_file				):
           copyfile(self.spm_file       ,					_lowerCAmelCase				)
         elif not os.path.isfile(self.spm_file				):
           with open(_lowerCAmelCase       ,					"wb"				) as fi:
             lowercase    :Optional[Any]      	=						self.sp_model.serialized_model_proto()
             fi.write(_lowerCAmelCase				)

         return (str(_lowerCAmelCase				), str(_lowerCAmelCase				))






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]       ,					_lowerCAmelCase:					List[str]       ,					_lowerCAmelCase:					str = "en"       ,					_lowerCAmelCase:					Optional[List[str]] = None       ,					_lowerCAmelCase:					str = "ro"       ,					**_lowerCAmelCase:					List[str]       ,					):
         lowercase    :Optional[int]      	=						src_lang
         lowercase    :Optional[Any]      	=						tgt_lang
         self.set_src_lang_special_tokens(self.src_lang				)
         return super().prepare_seqaseq_batch(_lowerCAmelCase       ,					_lowerCAmelCase       ,					**_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					Optional[int]       ,					_lowerCAmelCase:					Optional[str]       ,					_lowerCAmelCase:					Optional[str]       ,					**_lowerCAmelCase:					Union[str, Any]				):
         if src_lang is None or tgt_lang is None:
           raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"				)
         lowercase    :Tuple      	=						src_lang
         lowercase    :List[str]      	=						self(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase       ,					**_lowerCAmelCase				)
         lowercase    :List[Any]      	=						self.get_lang_id(_lowerCAmelCase				)
         lowercase    :Optional[int]      	=						tgt_lang_id
         return inputs






       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         self.set_src_lang_special_tokens(self.src_lang				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         self.set_tgt_lang_special_tokens(self.tgt_lang				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]       ,					_lowerCAmelCase:					str				):
         lowercase    :int      	=						self.get_lang_token(_lowerCAmelCase				)
         lowercase    :Union[str, Any]      	=						self.lang_token_to_id[lang_token]
         lowercase    :Dict      	=						[self.cur_lang_id]
         lowercase    :int      	=						[self.eos_token_id]






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]       ,					_lowerCAmelCase:					str				):
         lowercase    :Optional[int]      	=						self.get_lang_token(_lowerCAmelCase				)
         lowercase    :List[Any]      	=						self.lang_token_to_id[lang_token]
         lowercase    :str      	=						[self.cur_lang_id]
         lowercase    :List[str]      	=						[self.eos_token_id]






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]       ,					_lowerCAmelCase:					str				):
         return self.lang_code_to_token[lang]






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]       ,					_lowerCAmelCase:					str				):
         lowercase    :Dict      	=						self.get_lang_token(_lowerCAmelCase				)
         return self.lang_token_to_id[lang_token]







def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  lowercase    :Optional[int]      	=						sentencepiece.SentencePieceProcessor(**lowerCamelCase   )
  spm.Load(str(lowerCamelCase   )   )
  return spm



def       UpperCAmelCase__  (     lowerCamelCase   ):
  with open(lowerCamelCase,			"r"   ) as f:
    return json.load(lowerCamelCase   )







def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  with open(lowerCamelCase,			"w"   ) as f:
    json.dump(lowerCamelCase,			lowerCamelCase,			indent=2   )










def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase   ):
  if discount_rate < 0:
    raise ValueError("Discount rate cannot be negative"   )
  if not cash_flows:
    raise ValueError("Cash flows list cannot be empty"   )
  lowercase    :Dict      	=						sum(
      cash_flow / ((1 + discount_rate) ** i) for i, cash_flow in enumerate(lowerCamelCase   )   )
  return round(lowerCamelCase,			ndigits=2   )


if __name__ == "__main__":
     import doctest

     doctest.testmod()



