



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


a_						:     Optional[Any]	     =		logging.get_logger(__name__)

a_						:     List[str]	     =		'▁'

a_						:     str	     =		{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

a_						:     Optional[int]	     =		{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

a_						:     List[Any]	     =		{
    'facebook/m2m100_418M': 1_0_2_4,
}

# fmt: off
a_						:     Tuple	     =		{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}





class    lowerCamelCase__  (						lowercase__):
     """simple docstring"""



     _A       =			VOCAB_FILES_NAMES
     _A       =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A       =			PRETRAINED_VOCAB_FILES_MAP
     _A       =			['input_ids', 'attention_mask']

     _A       =			[]
     _A       =			[]







     def __init__(self				,			__a				,			__a				,			__a=None				,			__a=None				,			__a="<s>"				,			__a="</s>"				,			__a="</s>"				,			__a="<pad>"				,			__a="<unk>"				,			__a="m2m100"				,			__a = None				,			__a=8				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		{} if sp_model_kwargs is None else sp_model_kwargs

           lowerCamelCase								=		language_codes
           lowerCamelCase								=		FAIRSEQ_LANGUAGE_CODES[language_codes]
           lowerCamelCase								=		{lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

           lowerCamelCase								=		kwargs.get("additional_special_tokens"				,			[]    )
           kwargs["additional_special_tokens"] += [
               self.get_lang_token(UpperCAmelCase__    )
               for lang_code in fairseq_language_code
               if self.get_lang_token(UpperCAmelCase__    ) not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               src_lang=UpperCAmelCase__				,			tgt_lang=UpperCAmelCase__				,			bos_token=UpperCAmelCase__				,			eos_token=UpperCAmelCase__				,			sep_token=UpperCAmelCase__				,			unk_token=UpperCAmelCase__				,			pad_token=UpperCAmelCase__				,			language_codes=UpperCAmelCase__				,			sp_model_kwargs=self.sp_model_kwargs				,			num_madeup_words=UpperCAmelCase__				,			**UpperCAmelCase__				,			)

           lowerCamelCase								=		vocab_file
           lowerCamelCase								=		load_json(UpperCAmelCase__    )
           lowerCamelCase								=		{v: k for k, v in self.encoder.items()}
           lowerCamelCase								=		spm_file
           lowerCamelCase								=		load_spm(UpperCAmelCase__				,			self.sp_model_kwargs    )

           lowerCamelCase								=		len(self.encoder    )

           lowerCamelCase								=		{
               self.get_lang_token(UpperCAmelCase__    ): self.encoder_size + i for i, lang_code in enumerate(UpperCAmelCase__    )
           }
           lowerCamelCase								=		{lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCAmelCase__    )}
           lowerCamelCase								=		{v: k for k, v in self.lang_token_to_id.items()}

           lowerCamelCase								=		src_lang if src_lang is not None else '''en'''
           lowerCamelCase								=		tgt_lang
           lowerCamelCase								=		self.get_lang_id(self._src_lang    )
           self.set_src_lang_special_tokens(self._src_lang    )

           lowerCamelCase								=		num_madeup_words







     @property
     def 						_a      (self    ):






           '''simple docstring'''



           return len(self.encoder    ) + len(self.lang_token_to_id    )







     @property
     def 						_a      (self    ):






           '''simple docstring'''



           return self._src_lang







     @src_lang.setter
     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		new_src_lang
           self.set_src_lang_special_tokens(self._src_lang    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.encode(UpperCAmelCase__				,			out_type=UpperCAmelCase__    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           if token in self.lang_token_to_id:
                 return self.lang_token_to_id[token]
           return self.encoder.get(UpperCAmelCase__				,			self.encoder[self.unk_token]    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           if index in self.id_to_lang_token:
                 return self.id_to_lang_token[index]
           return self.decoder.get(UpperCAmelCase__				,			self.unk_token    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		[]
           lowerCamelCase								=		''''''
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       out_string += self.sp_model.decode(UpperCAmelCase__    ) + token
                       lowerCamelCase								=		[]
                 else:
                       current_sub_tokens.append(UpperCAmelCase__    )
           out_string += self.sp_model.decode(UpperCAmelCase__    )
           return out_string.strip()







     def 						_a      (self				,			__a				,			__a = None				,			__a = False    ):






           '''simple docstring'''



           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=UpperCAmelCase__				,			token_ids_a=UpperCAmelCase__				,			already_has_special_tokens=UpperCAmelCase__    )

           lowerCamelCase								=		[1] * len(self.prefix_tokens    )
           lowerCamelCase								=		[1] * len(self.suffix_tokens    )
           if token_ids_a is None:
                 return prefix_ones + ([0] * len(UpperCAmelCase__    )) + suffix_ones
           return prefix_ones + ([0] * len(UpperCAmelCase__    )) + ([0] * len(UpperCAmelCase__    )) + suffix_ones







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		{self.convert_ids_to_tokens(UpperCAmelCase__    ): i for i in range(self.vocab_size    )}
           vocab.update(self.added_tokens_encoder    )
           return vocab







     def __getstate__(self    ):






           '''simple docstring'''



           lowerCamelCase								=		self.__dict__.copy()
           lowerCamelCase								=		None
           return state







     def __setstate__(self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		d

           # for backward compatibility
           if not hasattr(self				,			"sp_model_kwargs"    ):
                 lowerCamelCase								=		{}

           lowerCamelCase								=		load_spm(self.spm_file				,			self.sp_model_kwargs    )







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		Path(UpperCAmelCase__    )
           if not save_dir.is_dir():
                 raise OSError(F"""{save_directory} should be a directory"""    )
           lowerCamelCase								=		save_dir / (
               (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
           )
           lowerCamelCase								=		save_dir / (
               (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
           )

           save_json(self.encoder				,			UpperCAmelCase__    )

           if os.path.abspath(self.spm_file    ) != os.path.abspath(UpperCAmelCase__    ) and os.path.isfile(self.spm_file    ):
                 copyfile(self.spm_file				,			UpperCAmelCase__    )
           elif not os.path.isfile(self.spm_file    ):
                 with open(UpperCAmelCase__				,			"wb"    ) as fi:
                       lowerCamelCase								=		self.sp_model.serialized_model_proto()
                       fi.write(UpperCAmelCase__    )

           return (str(UpperCAmelCase__    ), str(UpperCAmelCase__    ))







     def 						_a      (self				,			__a				,			__a = "en"				,			__a = None				,			__a = "ro"				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		src_lang
           lowerCamelCase								=		tgt_lang
           self.set_src_lang_special_tokens(self.src_lang    )
           return super().prepare_seqaseq_batch(UpperCAmelCase__				,			UpperCAmelCase__				,			**UpperCAmelCase__    )







     def 						_a      (self				,			__a				,			__a				,			__a				,			**__a    ):






           '''simple docstring'''



           if src_lang is None or tgt_lang is None:
                 raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"    )
           lowerCamelCase								=		src_lang
           lowerCamelCase								=		self(UpperCAmelCase__				,			add_special_tokens=UpperCAmelCase__				,			**UpperCAmelCase__    )
           lowerCamelCase								=		self.get_lang_id(UpperCAmelCase__    )
           lowerCamelCase								=		tgt_lang_id
           return inputs







     def 						_a      (self    ):






           '''simple docstring'''



           self.set_src_lang_special_tokens(self.src_lang    )







     def 						_a      (self    ):






           '''simple docstring'''



           self.set_tgt_lang_special_tokens(self.tgt_lang    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.get_lang_token(UpperCAmelCase__    )
           lowerCamelCase								=		self.lang_token_to_id[lang_token]
           lowerCamelCase								=		[self.cur_lang_id]
           lowerCamelCase								=		[self.eos_token_id]







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.get_lang_token(UpperCAmelCase__    )
           lowerCamelCase								=		self.lang_token_to_id[lang_token]
           lowerCamelCase								=		[self.cur_lang_id]
           lowerCamelCase								=		[self.eos_token_id]







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.lang_code_to_token[lang]







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.get_lang_token(UpperCAmelCase__    )
           return self.lang_token_to_id[lang_token]



def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		sentencepiece.SentencePieceProcessor(**UpperCAmelCase__ )
      spm.Load(str(UpperCAmelCase__ ) )
      return spm

def     __lowercase(	UpperCAmelCase__ ):


      """simple docstring"""



      with open(UpperCAmelCase__   ,   "r" ) as f:
            return json.load(UpperCAmelCase__ )






def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      with open(UpperCAmelCase__   ,   "w" ) as f:
            json.dump(UpperCAmelCase__   ,   UpperCAmelCase__   ,   indent=2 )




from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch





@require_torch
@require_tf
class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		SMALL_MODEL_IDENTIFIER
           lowerCamelCase								=		"pt"
           lowerCamelCase								=		"tf"







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		AutoModel.from_pretrained(self.test_model    )
           model_pt.save_pretrained(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		TFAutoModel.from_pretrained(self.test_model				,			from_pt=__a    )
           model_tf.save_pretrained(__a    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		"mock_framework"

           # Framework provided - return whatever the user provides
           lowerCamelCase								=		FeaturesManager.determine_framework(self.test_model				,			__a    )
           self.assertEqual(__a				,			__a    )

           # Local checkpoint and framework provided - return provided framework
           # PyTorch checkpoint
           with TemporaryDirectory() as local_pt_ckpt:
                 self._setup_pt_ckpt(__a    )
                 lowerCamelCase								=		FeaturesManager.determine_framework(__a				,			__a    )
                 self.assertEqual(__a				,			__a    )

           # TensorFlow checkpoint
           with TemporaryDirectory() as local_tf_ckpt:
                 self._setup_tf_ckpt(__a    )
                 lowerCamelCase								=		FeaturesManager.determine_framework(__a				,			__a    )
                 self.assertEqual(__a				,			__a    )







     def 						_a      (self    ):






           '''simple docstring'''



           with TemporaryDirectory() as local_pt_ckpt:
                 self._setup_pt_ckpt(__a    )
                 lowerCamelCase								=		FeaturesManager.determine_framework(__a    )
                 self.assertEqual(__a				,			self.framework_pt    )

           # TensorFlow checkpoint
           with TemporaryDirectory() as local_tf_ckpt:
                 self._setup_tf_ckpt(__a    )
                 lowerCamelCase								=		FeaturesManager.determine_framework(__a    )
                 self.assertEqual(__a				,			self.framework_tf    )

           # Invalid local checkpoint
           with TemporaryDirectory() as local_invalid_ckpt:
                 with self.assertRaises(__a    ):
                       lowerCamelCase								=		FeaturesManager.determine_framework(__a    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		MagicMock(return_value=__a    )
           with patch("transformers.onnx.features.is_tf_available"				,			__a    ):
                 lowerCamelCase								=		FeaturesManager.determine_framework(self.test_model    )
                 self.assertEqual(__a				,			self.framework_pt    )

           # PyTorch not in environment -> use TensorFlow
           lowerCamelCase								=		MagicMock(return_value=__a    )
           with patch("transformers.onnx.features.is_torch_available"				,			__a    ):
                 lowerCamelCase								=		FeaturesManager.determine_framework(self.test_model    )
                 self.assertEqual(__a				,			self.framework_tf    )

           # Both in environment -> use PyTorch
           lowerCamelCase								=		MagicMock(return_value=__a    )
           lowerCamelCase								=		MagicMock(return_value=__a    )
           with patch("transformers.onnx.features.is_tf_available"				,			__a    ), patch(
               "transformers.onnx.features.is_torch_available"				,			__a    ):
                 lowerCamelCase								=		FeaturesManager.determine_framework(self.test_model    )
                 self.assertEqual(__a				,			self.framework_pt    )

           # Both not in environment -> raise error
           lowerCamelCase								=		MagicMock(return_value=__a    )
           lowerCamelCase								=		MagicMock(return_value=__a    )
           with patch("transformers.onnx.features.is_tf_available"				,			__a    ), patch(
               "transformers.onnx.features.is_torch_available"				,			__a    ):
                 with self.assertRaises(__a    ):
                       lowerCamelCase								=		FeaturesManager.determine_framework(self.test_model    )