






import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase      :    Optional[Any]   		=   get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
	from transformers.models.plbart.modeling_plbart import shift_tokens_right

__lowerCamelCase      :    str   		=   5_0003
__lowerCamelCase      :    List[Any]   		=   5_0002



@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE__		(       UpperCamelCase_				,     unittest.TestCase		):





					"""simple docstring"""






					a_			=	PLBartTokenizer
					a_			=	None
					a_			=	False



					def  _lowercase  (      self  :						int ):
						super().setUp()

						# We have a SentencePiece fixture for testing
						snake_case__				:   List[Any]     =						PLBartTokenizer(__A							,	language_codes="base"							,	keep_accents=__A )
						tokenizer.save_pretrained(self.tmpdirname )



					def  _lowercase  (      self  :						Tuple ):
						snake_case__				:   Tuple     =						PLBartTokenizer(__A							,	language_codes="base"							,	keep_accents=__A )

						snake_case__				:   Optional[int]     =						tokenizer.tokenize("This is a test" )
						self.assertListEqual(__A							,	["▁This", "▁is", "▁a", "▁t", "est"] )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__A )							,	[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,	)

						snake_case__				:   Optional[Any]     =						tokenizer.tokenize("I was born in 92000, and this is falsé." )
						self.assertListEqual(
						    __A							,	[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "9",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "é",
						        ".",
						    ]							,	)
						snake_case__				:   Optional[int]     =						tokenizer.convert_tokens_to_ids(__A )
						self.assertListEqual(
						    __A							,	[
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
						    ]							,	)

						snake_case__				:   Union[str, Any]     =						tokenizer.convert_ids_to_tokens(__A )
						self.assertListEqual(
						    __A							,	[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "<unk>",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "<unk>",
						        ".",
						    ]							,	)

						snake_case__				:   int     =						tokenizer.vocab_size
						snake_case__				:   Tuple     =						[tokenizer.convert_ids_to_tokens(__A ) for x in range(end - 4							,	__A )]

						self.assertListEqual(__A							,	["__java__", "__python__", "__en_XX__", "<mask>"] )

						snake_case__				:   Tuple     =						"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
						snake_case__				:   Dict     =						tokenizer(__A ).input_ids
						self.assertEqual(
						    tokenizer.decode(__A							,	skip_special_tokens=__A							,	clean_up_tokenization_spaces=__A )							,	__A							,	)






					def  _lowercase  (      self  :						Dict ):
						snake_case__				:   Dict     =						PLBartTokenizer(__A							,	language_codes="multi"							,	keep_accents=__A )

						snake_case__				:   Optional[Any]     =						tokenizer.tokenize("This is a test" )
						self.assertListEqual(__A							,	["▁This", "▁is", "▁a", "▁t", "est"] )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__A )							,	[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,	)

						snake_case__				:   Dict     =						tokenizer.tokenize("I was born in 92000, and this is falsé." )
						self.assertListEqual(
						    __A							,	[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "9",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "é",
						        ".",
						    ]							,	)
						snake_case__				:   Dict     =						tokenizer.convert_tokens_to_ids(__A )
						self.assertListEqual(
						    __A							,	[
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
						    ]							,	)

						snake_case__				:   Optional[Any]     =						tokenizer.convert_ids_to_tokens(__A )
						self.assertListEqual(
						    __A							,	[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "<unk>",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "<unk>",
						        ".",
						    ]							,	)
						snake_case__				:   List[Any]     =						tokenizer.vocab_size
						snake_case__				:   Union[str, Any]     =						[tokenizer.convert_ids_to_tokens(__A ) for x in range(end - 7							,	__A )]

						self.assertListEqual(
						    __A							,	["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"] )
						snake_case__				:   str     =						"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
						snake_case__				:   List[Any]     =						tokenizer(__A ).input_ids
						self.assertEqual(
						    tokenizer.decode(__A							,	skip_special_tokens=__A							,	clean_up_tokenization_spaces=__A )							,	__A							,	)



@require_torch
@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE__		(       unittest.TestCase		):





					"""simple docstring"""






					a_			=	"uclanlp/plbart-python-en_XX"
					a_			=	[
					    "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])",
					    "def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])",
					]
					a_			=	[
					    "Returns the maximum value of a b c.",
					    "Sums the values of a b c.",
					]
					a_			=	[
					    1_3_4,
					    5_4_5_2,
					    3_3_4_6_0,
					    3_3_4_4_1,
					    3_3_4_6_3,
					    3_3_4_6_5,
					    3_3_4_6_3,
					    3_3_4_4_9,
					    9_8_8,
					    2_0,
					    3_3_4_5_6,
					    1_9,
					    3_3_4_5_6,
					    7_7_1,
					    3_9,
					    4_2_5_8,
					    8_8_9,
					    3_3_1_8,
					    3_3_4_4_1,
					    3_3_4_6_3,
					    3_3_4_6_5,
					    3_3_4_6_3,
					    3_3_4_4_9,
					    2_4_7_1,
					    2,
					    PYTHON_CODE,
					]



					@classmethod
					def  _lowercase  (      cls  :						Optional[int] ):
						snake_case__				:   PLBartTokenizer     =						PLBartTokenizer.from_pretrained(
						    cls.checkpoint_name							,	language_codes="base"							,	src_lang="python"							,	tgt_lang="en_XX" )
						snake_case__				:   Optional[Any]     =						1
						return cls



					def  _lowercase  (      self  :						Optional[Any] ):
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]							,	5_0_0_0_1 )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]							,	5_0_0_0_2 )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]							,	5_0_0_0_3 )



					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   str     =						self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
						self.assertListEqual(self.expected_src_tokens							,	__A )



					def  _lowercase  (      self  :						Optional[Any] ):
						self.assertIn(__A							,	self.tokenizer.all_special_ids )
						snake_case__				:   List[str]     =						[EN_CODE, 9_0_3_7, 3_3_4_4_2, 5_7, 7_5_2, 1_5_3, 1_4, 5_6, 1_8, 9, 2]
						snake_case__				:   List[str]     =						self.tokenizer.decode(__A							,	skip_special_tokens=__A )
						snake_case__				:   int     =						self.tokenizer.decode(generated_ids[1:]							,	skip_special_tokens=__A )
						self.assertEqual(__A							,	__A )
						self.assertNotIn(self.tokenizer.eos_token							,	__A )



					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   int     =						["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 2_0]
						self.assertIsInstance(src_text[0]							,	__A )
						snake_case__				:   Dict     =						1_0
						snake_case__				:   List[Any]     =						self.tokenizer(__A							,	max_length=__A							,	truncation=__A ).input_ids[0]
						self.assertEqual(ids[-2]							,	2 )
						self.assertEqual(ids[-1]							,	__A )
						self.assertEqual(len(__A )							,	__A )



					def  _lowercase  (      self  :						Optional[Any] ):
						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"] )							,	[5_0_0_0_4, 5_0_0_0_1] )



					def  _lowercase  (      self  :						Optional[int] ):
						snake_case__				:   str     =						tempfile.mkdtemp()
						snake_case__				:   List[Any]     =						self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(__A )
						snake_case__				:   List[Any]     =						PLBartTokenizer.from_pretrained(__A )
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,	__A )



					@require_torch
					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   Any     =						self.tokenizer(self.src_text							,	text_target=self.tgt_text							,	padding=__A							,	return_tensors="pt" )
						snake_case__				:   int     =						shift_tokens_right(batch["labels"]							,	self.tokenizer.pad_token_id )

						# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
						self.assertEqual(batch.input_ids[1][-2:].tolist()							,	[2, PYTHON_CODE] )
						self.assertEqual(batch.decoder_input_ids[1][0]							,	__A )
						self.assertEqual(batch.decoder_input_ids[1][-1]							,	2 )
						self.assertEqual(batch.labels[1][-2:].tolist()							,	[2, EN_CODE] )



					@require_torch
					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   Optional[int]     =						self.tokenizer(
						    self.src_text							,	text_target=self.tgt_text							,	padding=__A							,	truncation=__A							,	max_length=len(self.expected_src_tokens )							,	return_tensors="pt"							,	)
						snake_case__				:   Optional[Any]     =						shift_tokens_right(batch["labels"]							,	self.tokenizer.pad_token_id )

						self.assertIsInstance(__A							,	__A )

						self.assertEqual((2, 2_6)							,	batch.input_ids.shape )
						self.assertEqual((2, 2_6)							,	batch.attention_mask.shape )
						snake_case__				:   Optional[int]     =						batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens							,	__A )
						self.assertEqual(2							,	batch.decoder_input_ids[0, -1] )  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens							,	[] )
						self.assertEqual(self.tokenizer.suffix_tokens							,	[self.tokenizer.eos_token_id, PYTHON_CODE] )



					def  _lowercase  (      self  :						Any ):
						snake_case__				:   List[Any]     =						self.tokenizer(self.src_text							,	padding=__A							,	truncation=__A							,	max_length=3							,	return_tensors="pt" )
						snake_case__				:   str     =						self.tokenizer(
						    text_target=self.tgt_text							,	padding=__A							,	truncation=__A							,	max_length=1_0							,	return_tensors="pt" )
						snake_case__				:   int     =						targets["input_ids"]
						snake_case__				:   Tuple     =						shift_tokens_right(__A							,	self.tokenizer.pad_token_id )

						self.assertEqual(batch.input_ids.shape[1]							,	3 )
						self.assertEqual(batch.decoder_input_ids.shape[1]							,	1_0 )






					@require_torch
					def  _lowercase  (      self  :						Tuple ):
						snake_case__				:   str     =						self.tokenizer._build_translation_inputs(
						    "A test"							,	return_tensors="pt"							,	src_lang="en_XX"							,	tgt_lang="java" )

						self.assertEqual(
						    nested_simplify(__A )							,	{
						        # A, test, EOS, en_XX
						        "input_ids": [[1_5_0, 2_4_2, 2, 5_0_0_0_3]],
						        "attention_mask": [[1, 1, 1, 1]],
						        # java
						        "forced_bos_token_id": 5_0_0_0_1,
						    }							,	)











import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Value
from .base import TaskTemplate



@dataclass(frozen=UpperCamelCase_		)
class 							SCREAMING_SNAKE_CASE__		(       UpperCamelCase_		):





					"""simple docstring"""






					a_			=	field(default="text-classification"				,     metadata={"include_in_asdict_even_if_is_default": True}		)
					a_			=	Features({"text": Value("string"		)}		)
					a_			=	Features({"labels": ClassLabel}		)
					a_			=	"text"
					a_			=	"labels"



					def  _lowercase  (      self  :						Tuple							,	__A  :						List[Any] ):
						if self.label_column not in features:
							raise ValueError(f'''Column {self.label_column} is not present in features.''' )
						if not isinstance(features[self.label_column]							,	__A ):
							raise ValueError(f'''Column {self.label_column} is not a ClassLabel.''' )
						snake_case__				:   Any     =						copy.deepcopy(self )
						snake_case__				:   Optional[Any]     =						self.label_schema.copy()
						snake_case__				:   List[str]     =						features[self.label_column]
						snake_case__				:   Dict     =						label_schema
						return task_template






					@property
					def  _lowercase  (      self  :						Tuple ):
						return {
						    self.text_column: "text",
						    self.label_column: "labels",
						}



