



'''simple docstring'''


import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase							       =			get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
		from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowercase							       =			50_003
lowercase							       =			50_002







@require_sentencepiece
@require_tokenizers
class 	UpperCAmelCase	(		__a					,	unittest.TestCase):


	'''simple docstring'''




	__magic_name__					: Any  =      PLBartTokenizer
	__magic_name__					: List[Any]  =      None
	__magic_name__					: Tuple  =      False




	def       lowercase_  (   self) ->      str:



						"""simple docstring"""







						super().setUp()

						# We have a SentencePiece fixture for testing
						a_       		=PLBartTokenizer(UpperCamelCase__							,  language_codes="base"							,  keep_accents=UpperCamelCase__)
						tokenizer.save_pretrained(self.tmpdirname)




	def       lowercase_  (   self) ->      Optional[int]:



						"""simple docstring"""







						a_       		=PLBartTokenizer(UpperCamelCase__							,  language_codes="base"							,  keep_accents=UpperCamelCase__)

						a_       		=tokenizer.tokenize("This is a test")
						self.assertListEqual(UpperCamelCase__							,  ["▁This", "▁is", "▁a", "▁t", "est"])

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(UpperCamelCase__)							,  [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,  )

						a_       		=tokenizer.tokenize("I was born in 92000, and this is falsé.")
						self.assertListEqual(
						    UpperCamelCase__							,  [
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "9",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "é",
						        ".",
						    ]							,  )
						a_       		=tokenizer.convert_tokens_to_ids(UpperCamelCase__)
						self.assertListEqual(
						    UpperCamelCase__							,  [
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
						    ]							,  )

						a_       		=tokenizer.convert_ids_to_tokens(UpperCamelCase__)
						self.assertListEqual(
						    UpperCamelCase__							,  [
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "<unk>",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "<unk>",
						        ".",
						    ]							,  )

						a_       		=tokenizer.vocab_size
						a_       		=[tokenizer.convert_ids_to_tokens(UpperCamelCase__) for x in range(end - 4							,  UpperCamelCase__)]

						self.assertListEqual(UpperCamelCase__							,  ["__java__", "__python__", "__en_XX__", "<mask>"])

						a_       		="java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
						a_       		=tokenizer(UpperCamelCase__).input_ids
						self.assertEqual(
						    tokenizer.decode(UpperCamelCase__							,  skip_special_tokens=UpperCamelCase__							,  clean_up_tokenization_spaces=UpperCamelCase__)							,  UpperCamelCase__							,  )




	def       lowercase_  (   self) ->      Tuple:



						"""simple docstring"""







						a_       		=PLBartTokenizer(UpperCamelCase__							,  language_codes="multi"							,  keep_accents=UpperCamelCase__)

						a_       		=tokenizer.tokenize("This is a test")
						self.assertListEqual(UpperCamelCase__							,  ["▁This", "▁is", "▁a", "▁t", "est"])

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(UpperCamelCase__)							,  [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,  )

						a_       		=tokenizer.tokenize("I was born in 92000, and this is falsé.")
						self.assertListEqual(
						    UpperCamelCase__							,  [
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "9",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "é",
						        ".",
						    ]							,  )
						a_       		=tokenizer.convert_tokens_to_ids(UpperCamelCase__)
						self.assertListEqual(
						    UpperCamelCase__							,  [
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
						    ]							,  )

						a_       		=tokenizer.convert_ids_to_tokens(UpperCamelCase__)
						self.assertListEqual(
						    UpperCamelCase__							,  [
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "<unk>",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "<unk>",
						        ".",
						    ]							,  )
						a_       		=tokenizer.vocab_size
						a_       		=[tokenizer.convert_ids_to_tokens(UpperCamelCase__) for x in range(end - 7							,  UpperCamelCase__)]

						self.assertListEqual(
						    UpperCamelCase__							,  ["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"])
						a_       		="java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
						a_       		=tokenizer(UpperCamelCase__).input_ids
						self.assertEqual(
						    tokenizer.decode(UpperCamelCase__							,  skip_special_tokens=UpperCamelCase__							,  clean_up_tokenization_spaces=UpperCamelCase__)							,  UpperCamelCase__							,  )







@require_torch
@require_sentencepiece
@require_tokenizers
class 	UpperCAmelCase	(		unittest.TestCase):


	'''simple docstring'''




	__magic_name__					: Optional[int]  =      "uclanlp/plbart-python-en_XX"
	__magic_name__					: Dict  =      [
	    "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])",
	    "def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])",
	]
	__magic_name__					: int  =      [
	    "Returns the maximum value of a b c.",
	    "Sums the values of a b c.",
	]
	__magic_name__					: Union[str, Any]  =      [
	    134,
	    5_452,
	    33_460,
	    33_441,
	    33_463,
	    33_465,
	    33_463,
	    33_449,
	    988,
	    20,
	    33_456,
	    19,
	    33_456,
	    771,
	    39,
	    4_258,
	    889,
	    3_318,
	    33_441,
	    33_463,
	    33_465,
	    33_463,
	    33_449,
	    2_471,
	    2,
	    PYTHON_CODE,
	]




	@classmethod
	def       lowercase_  (   cls) ->      Optional[int]:



						"""simple docstring"""







						a_       		=PLBartTokenizer.from_pretrained(
						    cls.checkpoint_name							,  language_codes="base"							,  src_lang="python"							,  tgt_lang="en_XX")
						a_       		=1
						return cls




	def       lowercase_  (   self) ->      int:



						"""simple docstring"""







						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]							,  5_0_0_0_1)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]							,  5_0_0_0_2)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]							,  5_0_0_0_3)




	def       lowercase_  (   self) ->      Optional[Any]:



						"""simple docstring"""







						a_       		=self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
						self.assertListEqual(self.expected_src_tokens							,  UpperCamelCase__)




	def       lowercase_  (   self) ->      Any:



						"""simple docstring"""







						self.assertIn(UpperCamelCase__							,  self.tokenizer.all_special_ids)
						a_       		=[EN_CODE, 9_0_3_7, 3_3_4_4_2, 5_7, 7_5_2, 1_5_3, 1_4, 5_6, 1_8, 9, 2]
						a_       		=self.tokenizer.decode(UpperCamelCase__							,  skip_special_tokens=UpperCamelCase__)
						a_       		=self.tokenizer.decode(generated_ids[1:]							,  skip_special_tokens=UpperCamelCase__)
						self.assertEqual(UpperCamelCase__							,  UpperCamelCase__)
						self.assertNotIn(self.tokenizer.eos_token							,  UpperCamelCase__)




	def       lowercase_  (   self) ->      Any:



						"""simple docstring"""







						a_       		=["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 2_0]
						self.assertIsInstance(src_text[0]							,  UpperCamelCase__)
						a_       		=1_0
						a_       		=self.tokenizer(UpperCamelCase__							,  max_length=UpperCamelCase__							,  truncation=UpperCamelCase__).input_ids[0]
						self.assertEqual(ids[-2]							,  2)
						self.assertEqual(ids[-1]							,  UpperCamelCase__)
						self.assertEqual(len(UpperCamelCase__)							,  UpperCamelCase__)




	def       lowercase_  (   self) ->      Optional[Any]:



						"""simple docstring"""







						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"])							,  [5_0_0_0_4, 5_0_0_0_1])




	def       lowercase_  (   self) ->      Union[str, Any]:



						"""simple docstring"""







						a_       		=tempfile.mkdtemp()
						a_       		=self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(UpperCamelCase__)
						a_       		=PLBartTokenizer.from_pretrained(UpperCamelCase__)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,  UpperCamelCase__)




	@require_torch
	def       lowercase_  (   self) ->      Optional[int]:



						"""simple docstring"""







						a_       		=self.tokenizer(self.src_text							,  text_target=self.tgt_text							,  padding=UpperCamelCase__							,  return_tensors="pt")
						a_       		=shift_tokens_right(batch["labels"]							,  self.tokenizer.pad_token_id)

						# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
						self.assertEqual(batch.input_ids[1][-2:].tolist()							,  [2, PYTHON_CODE])
						self.assertEqual(batch.decoder_input_ids[1][0]							,  UpperCamelCase__)
						self.assertEqual(batch.decoder_input_ids[1][-1]							,  2)
						self.assertEqual(batch.labels[1][-2:].tolist()							,  [2, EN_CODE])




	@require_torch
	def       lowercase_  (   self) ->      int:



						"""simple docstring"""







						a_       		=self.tokenizer(
						    self.src_text							,  text_target=self.tgt_text							,  padding=UpperCamelCase__							,  truncation=UpperCamelCase__							,  max_length=len(self.expected_src_tokens)							,  return_tensors="pt"							,  )
						a_       		=shift_tokens_right(batch["labels"]							,  self.tokenizer.pad_token_id)

						self.assertIsInstance(UpperCamelCase__							,  UpperCamelCase__)

						self.assertEqual((2, 2_6)							,  batch.input_ids.shape)
						self.assertEqual((2, 2_6)							,  batch.attention_mask.shape)
						a_       		=batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens							,  UpperCamelCase__)
						self.assertEqual(2							,  batch.decoder_input_ids[0, -1])  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens							,  [])
						self.assertEqual(self.tokenizer.suffix_tokens							,  [self.tokenizer.eos_token_id, PYTHON_CODE])




	def       lowercase_  (   self) ->      Union[str, Any]:



						"""simple docstring"""







						a_       		=self.tokenizer(self.src_text							,  padding=UpperCamelCase__							,  truncation=UpperCamelCase__							,  max_length=3							,  return_tensors="pt")
						a_       		=self.tokenizer(
						    text_target=self.tgt_text							,  padding=UpperCamelCase__							,  truncation=UpperCamelCase__							,  max_length=1_0							,  return_tensors="pt")
						a_       		=targets["input_ids"]
						a_       		=shift_tokens_right(UpperCamelCase__							,  self.tokenizer.pad_token_id)

						self.assertEqual(batch.input_ids.shape[1]							,  3)
						self.assertEqual(batch.decoder_input_ids.shape[1]							,  1_0)




	@require_torch
	def       lowercase_  (   self) ->      List[Any]:



						"""simple docstring"""







						a_       		=self.tokenizer._build_translation_inputs(
						    "A test"							,  return_tensors="pt"							,  src_lang="en_XX"							,  tgt_lang="java")

						self.assertEqual(
						    nested_simplify(UpperCamelCase__)							,  {
						        # A, test, EOS, en_XX
						        "input_ids": [[1_5_0, 2_4_2, 2, 5_0_0_0_3]],
						        "attention_mask": [[1, 1, 1, 1]],
						        # java
						        "forced_bos_token_id": 5_0_0_0_1,
						    }							,  )







'''simple docstring'''


from __future__ import annotations






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     a_       		=str(lowercase__       )
     return len(lowercase__       ) == 9 and set(lowercase__       ) == set("123456789"       )






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     for base_num in range(9_9_9_9       ,	4_9_9_9       ,	-1       ):
          a_       		=1_0_0_0_0_2 * base_num
          if is_9_pandigital(lowercase__       ):
               return candidate

     for base_num in range(3_3_3       ,	9_9       ,	-1       ):
          a_       		=1_0_0_2_0_0_3 * base_num
          if is_9_pandigital(lowercase__       ):
               return candidate

     return None


if __name__ == "__main__":
  print(F"""{solution() = }""")


