
'''simple docstring'''

import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case:  str								=			get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
    from transformers.models.plbart.modeling_plbart import shift_tokens_right

__snake_case:  Any								=			5_00_03
__snake_case:  Any								=			5_00_02


@require_sentencepiece
@require_tokenizers
class  _UpperCAmelCase ( _snake_case	,unittest.TestCase    ):

     """simple docstring"""

     a_    	=					PLBartTokenizer
     a_    	=					None
     a_    	=					False


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       super().setUp()

       # We have a SentencePiece fixture for testing
       a_      :	List[str]      =							PLBartTokenizer(lowerCAmelCase_							,		language_codes="""base"""							,		keep_accents=lowerCAmelCase_ )
       tokenizer.save_pretrained(self.tmpdirname )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							PLBartTokenizer(lowerCAmelCase_							,		language_codes="""base"""							,		keep_accents=lowerCAmelCase_ )

       a_      :	Tuple      =							tokenizer.tokenize("""This is a test""" )
       self.assertListEqual(lowerCAmelCase_							,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )							,		[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]							,		)

       a_      :	List[Any]      =							tokenizer.tokenize("""I was born in 92000, and this is falsé.""" )
       self.assertListEqual(
           lowerCAmelCase_							,		[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """9""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """é""",
               """.""",
           ]							,		)
       a_      :	str      =							tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )
       self.assertListEqual(
           lowerCAmelCase_							,		[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]							,		)

       a_      :	Any      =							tokenizer.convert_ids_to_tokens(lowerCAmelCase_ )
       self.assertListEqual(
           lowerCAmelCase_							,		[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """<unk>""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """<unk>""",
               """.""",
           ]							,		)

       a_      :	Union[str, Any]      =							tokenizer.vocab_size
       a_      :	Union[str, Any]      =							[tokenizer.convert_ids_to_tokens(lowerCAmelCase_ ) for x in range(end - 4							,		lowerCAmelCase_ )]

       self.assertListEqual(lowerCAmelCase_							,		["""__java__""", """__python__""", """__en_XX__""", """<mask>"""] )

       a_      :	str      =							"""java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
       a_      :	Optional[Any]      =							tokenizer(lowerCAmelCase_ ).input_ids
       self.assertEqual(
           tokenizer.decode(lowerCAmelCase_							,		skip_special_tokens=lowerCAmelCase_							,		clean_up_tokenization_spaces=lowerCAmelCase_ )							,		lowerCAmelCase_							,		)






     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	str      =							PLBartTokenizer(lowerCAmelCase_							,		language_codes="""multi"""							,		keep_accents=lowerCAmelCase_ )

       a_      :	str      =							tokenizer.tokenize("""This is a test""" )
       self.assertListEqual(lowerCAmelCase_							,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )							,		[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]							,		)

       a_      :	Dict      =							tokenizer.tokenize("""I was born in 92000, and this is falsé.""" )
       self.assertListEqual(
           lowerCAmelCase_							,		[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """9""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """é""",
               """.""",
           ]							,		)
       a_      :	int      =							tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )
       self.assertListEqual(
           lowerCAmelCase_							,		[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]							,		)

       a_      :	Dict      =							tokenizer.convert_ids_to_tokens(lowerCAmelCase_ )
       self.assertListEqual(
           lowerCAmelCase_							,		[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """<unk>""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """<unk>""",
               """.""",
           ]							,		)
       a_      :	List[str]      =							tokenizer.vocab_size
       a_      :	int      =							[tokenizer.convert_ids_to_tokens(lowerCAmelCase_ ) for x in range(end - 7							,		lowerCAmelCase_ )]

       self.assertListEqual(
           lowerCAmelCase_							,		["""__java__""", """__python__""", """__en_XX__""", """__javascript__""", """__php__""", """__ruby__""", """__go__"""] )
       a_      :	List[Any]      =							"""java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
       a_      :	Union[str, Any]      =							tokenizer(lowerCAmelCase_ ).input_ids
       self.assertEqual(
           tokenizer.decode(lowerCAmelCase_							,		skip_special_tokens=lowerCAmelCase_							,		clean_up_tokenization_spaces=lowerCAmelCase_ )							,		lowerCAmelCase_							,		)







@require_torch
@require_sentencepiece
@require_tokenizers
class  _UpperCAmelCase ( unittest.TestCase    ):

     """simple docstring"""

     a_    	=					'uclanlp/plbart-python-en_XX'
     a_    	=					[
         'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
         'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
     ]
     a_    	=					[
         'Returns the maximum value of a b c.',
         'Sums the values of a b c.',
     ]
     a_    	=					[
         134,
         5_452,
         33_460,
         33_441,
         33_463,
         33_465,
         33_463,
         33_449,
         988,
         20,
         33_456,
         19,
         33_456,
         771,
         39,
         4_258,
         889,
         3_318,
         33_441,
         33_463,
         33_465,
         33_463,
         33_449,
         2_471,
         2,
         PYTHON_CODE,
     ]


     @classmethod
     def 							_lowerCAmelCase  (      cls ):

       '''simple docstring'''

       a_      :	List[str]      =							PLBartTokenizer.from_pretrained(
           cls.checkpoint_name							,		language_codes="""base"""							,		src_lang="""python"""							,		tgt_lang="""en_XX""" )
       a_      :	Any      =							1
       return cls


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__java__"""]							,		5_00_01 )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__python__"""]							,		5_00_02 )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__en_XX__"""]							,		5_00_03 )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Tuple      =							self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
       self.assertListEqual(self.expected_src_tokens							,		lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       self.assertIn(lowerCAmelCase_							,		self.tokenizer.all_special_ids )
       a_      :	Union[str, Any]      =							[EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
       a_      :	Optional[Any]      =							self.tokenizer.decode(lowerCAmelCase_							,		skip_special_tokens=lowerCAmelCase_ )
       a_      :	Any      =							self.tokenizer.decode(generated_ids[1:]							,		skip_special_tokens=lowerCAmelCase_ )
       self.assertEqual(lowerCAmelCase_							,		lowerCAmelCase_ )
       self.assertNotIn(self.tokenizer.eos_token							,		lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Tuple      =							["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 20]
       self.assertIsInstance(src_text[0]							,		lowerCAmelCase_ )
       a_      :	Optional[Any]      =							10
       a_      :	Any      =							self.tokenizer(lowerCAmelCase_							,		max_length=lowerCAmelCase_							,		truncation=lowerCAmelCase_ ).input_ids[0]
       self.assertEqual(ids[-2]							,		2 )
       self.assertEqual(ids[-1]							,		lowerCAmelCase_ )
       self.assertEqual(len(lowerCAmelCase_ )							,		lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """__java__"""] )							,		[5_00_04, 5_00_01] )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	str      =							tempfile.mkdtemp()
       a_      :	int      =							self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(lowerCAmelCase_ )
       a_      :	Optional[int]      =							PLBartTokenizer.from_pretrained(lowerCAmelCase_ )
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,		lowerCAmelCase_ )


     @require_torch
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.tokenizer(self.src_text							,		text_target=self.tgt_text							,		padding=lowerCAmelCase_							,		return_tensors="""pt""" )
       a_      :	str      =							shift_tokens_right(batch["""labels"""]							,		self.tokenizer.pad_token_id )

       # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
       self.assertEqual(batch.input_ids[1][-2:].tolist()							,		[2, PYTHON_CODE] )
       self.assertEqual(batch.decoder_input_ids[1][0]							,		lowerCAmelCase_ )
       self.assertEqual(batch.decoder_input_ids[1][-1]							,		2 )
       self.assertEqual(batch.labels[1][-2:].tolist()							,		[2, EN_CODE] )


     @require_torch
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.tokenizer(
           self.src_text							,		text_target=self.tgt_text							,		padding=lowerCAmelCase_							,		truncation=lowerCAmelCase_							,		max_length=len(self.expected_src_tokens )							,		return_tensors="""pt"""							,		)
       a_      :	Dict      =							shift_tokens_right(batch["""labels"""]							,		self.tokenizer.pad_token_id )

       self.assertIsInstance(lowerCAmelCase_							,		lowerCAmelCase_ )

       self.assertEqual((2, 26)							,		batch.input_ids.shape )
       self.assertEqual((2, 26)							,		batch.attention_mask.shape )
       a_      :	Union[str, Any]      =							batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens							,		lowerCAmelCase_ )
       self.assertEqual(2							,		batch.decoder_input_ids[0, -1] )  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens							,		[] )
       self.assertEqual(self.tokenizer.suffix_tokens							,		[self.tokenizer.eos_token_id, PYTHON_CODE] )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[int]      =							self.tokenizer(self.src_text							,		padding=lowerCAmelCase_							,		truncation=lowerCAmelCase_							,		max_length=3							,		return_tensors="""pt""" )
       a_      :	Dict      =							self.tokenizer(
           text_target=self.tgt_text							,		padding=lowerCAmelCase_							,		truncation=lowerCAmelCase_							,		max_length=10							,		return_tensors="""pt""" )
       a_      :	Union[str, Any]      =							targets["""input_ids"""]
       a_      :	Tuple      =							shift_tokens_right(lowerCAmelCase_							,		self.tokenizer.pad_token_id )

       self.assertEqual(batch.input_ids.shape[1]							,		3 )
       self.assertEqual(batch.decoder_input_ids.shape[1]							,		10 )






     @require_torch
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.tokenizer._build_translation_inputs(
           """A test"""							,		return_tensors="""pt"""							,		src_lang="""en_XX"""							,		tgt_lang="""java""" )

       self.assertEqual(
           nested_simplify(lowerCAmelCase_ )							,		{
               # A, test, EOS, en_XX
               """input_ids""": [[1_50, 2_42, 2, 5_00_03]],
               """attention_mask""": [[1, 1, 1, 1]],
               # java
               """forced_bos_token_id""": 5_00_01,
           }							,		)




'''simple docstring'''

import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate


@dataclass(frozen=lowerCAmelCase__    )
class  _UpperCAmelCase ( lowerCAmelCase__    ):

     """simple docstring"""

     a_    	=					field(default="automatic-speech-recognition"	,metadata={"include_in_asdict_even_if_is_default": True}    )
     a_    	=					Features({"audio": Audio()}    )
     a_    	=					Features({"transcription": Value("string"    )}    )
     a_    	=					"audio"
     a_    	=					"transcription"


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       if self.audio_column not in features:
         raise ValueError(f'''Column {self.audio_column} is not present in features.''' )
       if not isinstance(features[self.audio_column]							,		lowerCAmelCase_ ):
         raise ValueError(f'''Column {self.audio_column} is not an Audio type.''' )
       a_      :	int      =							copy.deepcopy(self )
       a_      :	Optional[int]      =							self.input_schema.copy()
       a_      :	Any      =							features[self.audio_column]
       a_      :	Any      =							input_schema
       return task_template






     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return {self.audio_column: "audio", self.transcription_column: "transcription"}


