
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
    from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
    __lowercase											=      get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
    from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowercase											=      12_8022
__lowercase											=      12_8028


@require_sentencepiece
class 			lowerCamelCase_  (     UpperCAmelCase_		,			unittest.TestCase				):






    '''simple docstring'''






    a__			:		Optional[int]      			=   MaMaaaTokenizer
    a__			:		Optional[Any]      			=   False
    a__			:		Dict      			=   False
    a__			:		Tuple      			=   True







    def     UpperCamelCase__     (			self) ->			Any:
        super().setUp()

        __UpperCamelCase  :Optional[int]    						=	['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
        __UpperCamelCase  :Tuple    						=	dict(zip(__lowercase				,				range(len(__lowercase))))
        __UpperCamelCase  :Optional[Any]    						=	Path(self.tmpdirname)
        save_json(__lowercase				,				save_dir / VOCAB_FILES_NAMES['''vocab_file'''])
        if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
            copyfile(__lowercase				,				save_dir / VOCAB_FILES_NAMES['''spm_file'''])

        __UpperCamelCase  :str    						=	MaMaaaTokenizer.from_pretrained(self.tmpdirname)
        tokenizer.save_pretrained(self.tmpdirname)







    def     UpperCamelCase__     (			self				,				**__lowercase) ->			List[Any]:
        return MaMaaaTokenizer.from_pretrained(self.tmpdirname				,				**__lowercase)







    def     UpperCamelCase__     (			self				,				__lowercase) ->			Union[str, Any]:
        return (
            "This is a test",
            "This is a test",
        )







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        __UpperCamelCase  :Optional[int]    						=	'''</s>'''
        __UpperCamelCase  :Any    						=	0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase)				,				__lowercase)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase)				,				__lowercase)







    def     UpperCamelCase__     (			self) ->			int:
        __UpperCamelCase  :str    						=	self.get_tokenizer()
        __UpperCamelCase  :int    						=	list(tokenizer.get_vocab().keys())

        self.assertEqual(vocab_keys[0]				,				'''</s>''')
        self.assertEqual(vocab_keys[1]				,				'''<unk>''')
        self.assertEqual(vocab_keys[-1]				,				'''<s>''')
        self.assertEqual(len(__lowercase)				,				tokenizer.vocab_size + len(tokenizer.get_added_vocab()))







    @unittest.skip('''Skip this test while all models are still to be uploaded.''')
    def     UpperCamelCase__     (			self) ->			Union[str, Any]:
        pass







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        __UpperCamelCase  :List[str]    						=	self.get_tokenizer()

        __UpperCamelCase  :str    						=	tokenizer.tokenize('''This is a test''')
        self.assertListEqual(__lowercase				,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__lowercase)				,				[2, 3, 4, 5, 6]				,				)

        __UpperCamelCase  :Tuple    						=	tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6])
        self.assertListEqual(__lowercase				,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

        __UpperCamelCase  :List[str]    						=	tokenizer.convert_tokens_to_string(__lowercase)
        self.assertEqual(__lowercase				,				'''This is a test''')







    @slow
    def     UpperCamelCase__     (			self) ->			str:
        # fmt: off
        __UpperCamelCase  :List[str]    						=	{'''input_ids''': [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__lowercase				,				model_name='''facebook/m2m100_418M'''				,				revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e'''				,				)


@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase_  (     unittest.TestCase				):






    '''simple docstring'''






    a__			:		List[str]      			=   """facebook/m2m100_418M"""
    a__			:		Tuple      			=   [
        """In my opinion, there are two levels of response from the French government.""",
        """NSA Affair Emphasizes Complete Lack of Debate on Intelligence""",
    ]
    a__			:		Optional[int]      			=   [
        """Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.""",
        """L'affaire NSA souligne l'absence totale de débat sur le renseignement""",
    ]

    # fmt: off
    a__			:		Union[str, Any]      			=   [EN_CODE, 5_9_3, 1_9_4_9, 1_1_5_7_8_1, 4, 7_1_5_8_6, 4_2_3_4, 6_0_6_3_3, 1_2_6_2_3_3, 4_3_2, 1_2_3_8_0_8, 1_5_5_9_2, 1_1_9_7, 1_1_7_1_3_2, 1_2_0_6_1_8, 5, 2]







    @classmethod
    def     UpperCamelCase__     (			cls) ->			Union[str, Any]:
        __UpperCamelCase  :MaMaaaTokenizer    						=	MaMaaaTokenizer.from_pretrained(
            cls.checkpoint_name				,				src_lang='''en'''				,				tgt_lang='''fr''')
        __UpperCamelCase  :List[Any]    						=	1
        return cls







    def     UpperCamelCase__     (			self) ->			Any:
        self.assertEqual(self.tokenizer.get_lang_id('''ar''')				,				128_006)
        self.assertEqual(self.tokenizer.get_lang_id('''en''')				,				128_022)
        self.assertEqual(self.tokenizer.get_lang_id('''ro''')				,				128_076)
        self.assertEqual(self.tokenizer.get_lang_id('''mr''')				,				128_063)







    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :str    						=	self.tokenizer.get_vocab()
        self.assertEqual(len(__lowercase)				,				self.tokenizer.vocab_size)
        self.assertEqual(vocab['''<unk>''']				,				3)
        self.assertIn(self.tokenizer.get_lang_token('''en''')				,				__lowercase)







    def     UpperCamelCase__     (			self) ->			int:
        __UpperCamelCase  :Tuple    						=	'''en'''
        __UpperCamelCase  :List[str]    						=	self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
        self.assertListEqual(self.expected_src_tokens				,				__lowercase)







    def     UpperCamelCase__     (			self) ->			str:
        self.assertIn(__lowercase				,				self.tokenizer.all_special_ids)
        # fmt: off
        __UpperCamelCase  :Tuple    						=	[FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
        # fmt: on
        __UpperCamelCase  :Optional[Any]    						=	self.tokenizer.decode(__lowercase				,				skip_special_tokens=__lowercase)
        __UpperCamelCase  :Any    						=	self.tokenizer.decode(generated_ids[1:]				,				skip_special_tokens=__lowercase)
        self.assertEqual(__lowercase				,				__lowercase)
        self.assertNotIn(self.tokenizer.eos_token				,				__lowercase)







    def     UpperCamelCase__     (			self) ->			int:
        __UpperCamelCase  :int    						=	tempfile.mkdtemp()
        __UpperCamelCase  :Dict    						=	self.tokenizer.lang_token_to_id
        self.tokenizer.save_pretrained(__lowercase)
        __UpperCamelCase  :int    						=	MaMaaaTokenizer.from_pretrained(__lowercase)
        self.assertDictEqual(new_tok.lang_token_to_id				,				__lowercase)







    @require_torch
    def     UpperCamelCase__     (			self) ->			int:
        __UpperCamelCase  :Dict    						=	'''en'''
        __UpperCamelCase  :str    						=	'''fr'''

        __UpperCamelCase  :int    						=	self.tokenizer(self.src_text				,				text_target=self.tgt_text				,				padding=__lowercase				,				return_tensors='''pt''')

        __UpperCamelCase  :int    						=	shift_tokens_right(
            batch['''labels''']				,				self.tokenizer.pad_token_id				,				self.tokenizer.eos_token_id)

        for k in batch:
            __UpperCamelCase  :str    						=	batch[k].tolist()
        # batch = {k: v.tolist() for k,v in batch.items()}
        # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
        # batch.decoder_inputs_ids[0][0] ==
        assert batch.input_ids[1][0] == EN_CODE
        assert batch.input_ids[1][-1] == 2
        assert batch.labels[1][0] == FR_CODE
        assert batch.labels[1][-1] == 2
        assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]







    @require_torch
    def     UpperCamelCase__     (			self) ->			Any:
        __UpperCamelCase  :Any    						=	'''mr'''
        self.assertListEqual(self.tokenizer.prefix_tokens				,				[self.tokenizer.get_lang_id('''mr''')])
        self.assertListEqual(self.tokenizer.suffix_tokens				,				[self.tokenizer.eos_token_id])

        __UpperCamelCase  :str    						=	'''zh'''
        self.assertListEqual(self.tokenizer.prefix_tokens				,				[self.tokenizer.get_lang_id('''zh''')])
        self.assertListEqual(self.tokenizer.suffix_tokens				,				[self.tokenizer.eos_token_id])







    @require_torch
    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :Union[str, Any]    						=	'''mr'''
        self.tokenizer._switch_to_target_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens				,				[self.tokenizer.get_lang_id('''mr''')])
        self.assertListEqual(self.tokenizer.suffix_tokens				,				[self.tokenizer.eos_token_id])
        self.tokenizer._switch_to_input_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens				,				[self.tokenizer.get_lang_id(self.tokenizer.src_lang)])

        __UpperCamelCase  :Union[str, Any]    						=	'''zh'''
        self.tokenizer._switch_to_target_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens				,				[self.tokenizer.get_lang_id('''zh''')])
        self.assertListEqual(self.tokenizer.suffix_tokens				,				[self.tokenizer.eos_token_id])
        self.tokenizer._switch_to_input_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens				,				[self.tokenizer.get_lang_id(self.tokenizer.src_lang)])







    @require_torch
    def     UpperCamelCase__     (			self) ->			Optional[Any]:
        __UpperCamelCase  :str    						=	self.tokenizer._build_translation_inputs('''A test'''				,				return_tensors='''pt'''				,				src_lang='''en'''				,				tgt_lang='''ar''')

        self.assertEqual(
            nested_simplify(__lowercase)				,				{
                # en_XX, A, test, EOS
                '''input_ids''': [[128_022, 58, 4_183, 2]],
                '''attention_mask''': [[1, 1, 1, 1]],
                # ar_AR
                '''forced_bos_token_id''': 128_006,
            }				,				)






from typing import List

import datasets
from datasets.tasks import AudioClassification

from ..folder_based_builder import folder_based_builder


__lowercase											=      datasets.utils.logging.get_logger(__name__)


class 			lowerCamelCase_  (     folder_based_builder.FolderBasedBuilderConfig				):






    '''simple docstring'''






    a__			:		bool      			=   None
    a__			:		bool      			=   None


class 			lowerCamelCase_  (     folder_based_builder.FolderBasedBuilder				):






    '''simple docstring'''






    a__			:		List[Any]      			=   datasets.Audio()
    a__			:		int      			=   """audio"""
    a__			:		str      			=   AudioFolderConfig
    a__			:		List[str]  # definition at the bottom of the script
    a__			:		int      			=   AudioClassification(audio_column="""audio"""		,			label_column="""label"""				)
__lowercase											=      [
    '''.aiff''',
    '''.au''',
    '''.avr''',
    '''.caf''',
    '''.flac''',
    '''.htk''',
    '''.svx''',
    '''.mat4''',
    '''.mat5''',
    '''.mpc2k''',
    '''.ogg''',
    '''.paf''',
    '''.pvf''',
    '''.raw''',
    '''.rf64''',
    '''.sd2''',
    '''.sds''',
    '''.ircam''',
    '''.voc''',
    '''.w64''',
    '''.wav''',
    '''.nist''',
    '''.wavex''',
    '''.wve''',
    '''.xi''',
    '''.mp3''',
    '''.opus''',
]
__lowercase											=      AUDIO_EXTENSIONS





