



"""simple docstring"""





import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
  from cookiecutter.main import cookiecutter

  A      =		True
except ImportError:
  A      =		False

A      =		logging.get_logger(__name__)  # pylint: disable=invalid-name
def      a__			(   lowerCAmelCase__   ):
      return AddNewModelCommand(args.testing			,    args.testing_file			,    path=args.path   )


class    lowercase__     (					lowerCAmelCase__   ):







       '''simple docstring'''


       @staticmethod
       def   lowercase__       (							_UpperCAmelCase    :						Tuple					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							parser.add_parser("add-new-model"					)
             add_new_model_parser.add_argument("--testing"	,       action="store_true"	,       help="If in testing mode."					)
             add_new_model_parser.add_argument("--testing_file"	,       type=_SCREAMING_SNAKE_CASE	,       help="Configuration file on which to run."					)
             add_new_model_parser.add_argument(
                 "--path"	,       type=_SCREAMING_SNAKE_CASE	,       help="Path to cookiecutter. Should only be used for testing purposes."					)
             add_new_model_parser.set_defaults(func=_SCREAMING_SNAKE_CASE					)



       def __init__(							self    :						Dict	,       _UpperCAmelCase    :						Dict	,       _UpperCAmelCase    :						List[Any]	,       _UpperCAmelCase    :						Optional[Any]=None	,       *_UpperCAmelCase    :						List[Any]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							testing
             UpperCAmelCase_    				=							testing_file
             UpperCAmelCase_    				=							path







       def   lowercase__       (							self    :						Union[str, Any]					)				->		str:


             '''simple docstring'''




             warnings.warn(
                 "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
                 "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
                 "checks, you should use `transformers-cli add-new-model-like` instead."					)
             if not _has_cookiecutter:
                   raise ImportError(
                       "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
                       "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"					)
             # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
             UpperCAmelCase_    				=							[directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
             if len(_SCREAMING_SNAKE_CASE					) > 0:
                   raise ValueError(
                       "Several directories starting with `cookiecutter-template-` in current working directory. "
                       "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
                       "change your working directory."					)

             UpperCAmelCase_    				=							(
                 Path(_SCREAMING_SNAKE_CASE					).parent.parent.parent.parent if self._path is None else Path(self._path					).parent.parent
             )
             UpperCAmelCase_    				=							path_to_transformer_root / "templates" / "adding_a_new_model"

             # Execute cookiecutter
             if not self._testing:
                   cookiecutter(str(_SCREAMING_SNAKE_CASE					)					)
             else:
                   with open(self._testing_file	,       "r"					) as configuration_file:
                         UpperCAmelCase_    				=							json.load(_SCREAMING_SNAKE_CASE					)

                   cookiecutter(
                       str(path_to_cookiecutter if self._path is None else self._path					)	,       no_input=_SCREAMING_SNAKE_CASE	,       extra_context=_SCREAMING_SNAKE_CASE	,       )

             UpperCAmelCase_    				=							[directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]

             # Retrieve configuration
             with open(directory + "/configuration.json"	,       "r"					) as configuration_file:
                   UpperCAmelCase_    				=							json.load(_SCREAMING_SNAKE_CASE					)

             UpperCAmelCase_    				=							configuration["lowercase_modelname"]
             UpperCAmelCase_    				=							configuration["generate_tensorflow_pytorch_and_flax"]
             os.remove(F"""{directory}/configuration.json"""					)

             UpperCAmelCase_    				=							"PyTorch" in generate_tensorflow_pytorch_and_flax
             UpperCAmelCase_    				=							"TensorFlow" in generate_tensorflow_pytorch_and_flax
             UpperCAmelCase_    				=							"Flax" in generate_tensorflow_pytorch_and_flax

             UpperCAmelCase_    				=							F"""{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"""
             os.makedirs(_SCREAMING_SNAKE_CASE	,       exist_ok=_SCREAMING_SNAKE_CASE					)
             os.makedirs(F"""{path_to_transformer_root}/tests/models/{lowercase_model_name}"""	,       exist_ok=_SCREAMING_SNAKE_CASE					)

             # Tests require submodules as they have parent imports
             with open(F"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py"""	,       "w"					):
                   pass

             shutil.move(
                 F"""{directory}/__init__.py"""	,       F"""{model_dir}/__init__.py"""	,       )
             shutil.move(
                 F"""{directory}/configuration_{lowercase_model_name}.py"""	,       F"""{model_dir}/configuration_{lowercase_model_name}.py"""	,       )

             def remove_copy_lines(_UpperCAmelCase    :						str					):
                   with open(_SCREAMING_SNAKE_CASE	,       "r"					) as f:
                         UpperCAmelCase_    				=							f.readlines()
                   with open(_SCREAMING_SNAKE_CASE	,       "w"					) as f:
                         for line in lines:
                               if "# Copied from transformers." not in line:
                                     f.write(_SCREAMING_SNAKE_CASE					)

             if output_pytorch:
                   if not self._testing:
                         remove_copy_lines(F"""{directory}/modeling_{lowercase_model_name}.py"""					)

                   shutil.move(
                       F"""{directory}/modeling_{lowercase_model_name}.py"""	,       F"""{model_dir}/modeling_{lowercase_model_name}.py"""	,       )

                   shutil.move(
                       F"""{directory}/test_modeling_{lowercase_model_name}.py"""	,       F"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py"""	,       )
             else:
                   os.remove(F"""{directory}/modeling_{lowercase_model_name}.py"""					)
                   os.remove(F"""{directory}/test_modeling_{lowercase_model_name}.py"""					)

             if output_tensorflow:
                   if not self._testing:
                         remove_copy_lines(F"""{directory}/modeling_tf_{lowercase_model_name}.py"""					)

                   shutil.move(
                       F"""{directory}/modeling_tf_{lowercase_model_name}.py"""	,       F"""{model_dir}/modeling_tf_{lowercase_model_name}.py"""	,       )

                   shutil.move(
                       F"""{directory}/test_modeling_tf_{lowercase_model_name}.py"""	,       F"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py"""	,       )
             else:
                   os.remove(F"""{directory}/modeling_tf_{lowercase_model_name}.py"""					)
                   os.remove(F"""{directory}/test_modeling_tf_{lowercase_model_name}.py"""					)

             if output_flax:
                   if not self._testing:
                         remove_copy_lines(F"""{directory}/modeling_flax_{lowercase_model_name}.py"""					)

                   shutil.move(
                       F"""{directory}/modeling_flax_{lowercase_model_name}.py"""	,       F"""{model_dir}/modeling_flax_{lowercase_model_name}.py"""	,       )

                   shutil.move(
                       F"""{directory}/test_modeling_flax_{lowercase_model_name}.py"""	,       F"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py"""	,       )
             else:
                   os.remove(F"""{directory}/modeling_flax_{lowercase_model_name}.py"""					)
                   os.remove(F"""{directory}/test_modeling_flax_{lowercase_model_name}.py"""					)

             shutil.move(
                 F"""{directory}/{lowercase_model_name}.md"""	,       F"""{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md"""	,       )

             shutil.move(
                 F"""{directory}/tokenization_{lowercase_model_name}.py"""	,       F"""{model_dir}/tokenization_{lowercase_model_name}.py"""	,       )

             shutil.move(
                 F"""{directory}/tokenization_fast_{lowercase_model_name}.py"""	,       F"""{model_dir}/tokenization_{lowercase_model_name}_fast.py"""	,       )

             from os import fdopen, remove
             from shutil import copymode, move
             from tempfile import mkstemp

             def replace(_UpperCAmelCase    :						int	,       _UpperCAmelCase    :						Dict	,       _UpperCAmelCase    :						Any					):
                   # Create temp file
                   UpperCAmelCase_				,					UpperCAmelCase_    				=							mkstemp()
                   UpperCAmelCase_    				=							False
                   with fdopen(_SCREAMING_SNAKE_CASE	,       "w"					) as new_file:
                         with open(_SCREAMING_SNAKE_CASE					) as old_file:
                               for line in old_file:
                                     new_file.write(_SCREAMING_SNAKE_CASE					)
                                     if line_to_copy_below in line:
                                           UpperCAmelCase_    				=							True
                                           for line_to_copy in lines_to_copy:
                                                 new_file.write(_SCREAMING_SNAKE_CASE					)

                   if not line_found:
                         raise ValueError(F"""Line {line_to_copy_below} was not found in file."""					)

                   # Copy the file permissions from the old file to the new file
                   copymode(_SCREAMING_SNAKE_CASE	,       _SCREAMING_SNAKE_CASE					)
                   # Remove original file
                   remove(_SCREAMING_SNAKE_CASE					)
                   # Move new file
                   move(_SCREAMING_SNAKE_CASE	,       _SCREAMING_SNAKE_CASE					)

             def skip_units(_UpperCAmelCase    :						Any					):
                   return (
                       ("generating PyTorch" in line and not output_pytorch)
                       or ("generating TensorFlow" in line and not output_tensorflow)
                       or ("generating Flax" in line and not output_flax)
                   )

             def replace_in_files(_UpperCAmelCase    :						Optional[Any]					):
                   with open(_SCREAMING_SNAKE_CASE					) as datafile:
                         UpperCAmelCase_    				=							[]
                         UpperCAmelCase_    				=							False
                         UpperCAmelCase_    				=							False
                         for line in datafile:
                               if "# To replace in: " in line and "##" not in line:
                                     UpperCAmelCase_    				=							line.split("\""					)[1]
                                     UpperCAmelCase_    				=							skip_units(_SCREAMING_SNAKE_CASE					)
                               elif "# Below: " in line and "##" not in line:
                                     UpperCAmelCase_    				=							line.split("\""					)[1]
                                     UpperCAmelCase_    				=							skip_units(_SCREAMING_SNAKE_CASE					)
                               elif "# End." in line and "##" not in line:
                                     if not skip_file and not skip_snippet:
                                           replace(_SCREAMING_SNAKE_CASE	,       _SCREAMING_SNAKE_CASE	,       _SCREAMING_SNAKE_CASE					)

                                     UpperCAmelCase_    				=							[]
                               elif "# Replace with" in line and "##" not in line:
                                     UpperCAmelCase_    				=							[]
                               elif "##" not in line:
                                     lines_to_copy.append(_SCREAMING_SNAKE_CASE					)

                   remove(_SCREAMING_SNAKE_CASE					)

             replace_in_files(F"""{directory}/to_replace_{lowercase_model_name}.py"""					)
             os.rmdir(_SCREAMING_SNAKE_CASE					)









"""simple docstring"""





import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase      =		get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCamelCase      =		50_003
lowerCamelCase      =		50_002
@require_sentencepiece
@require_tokenizers
class    lowercase__     (					SCREAMING_SNAKE_CASE       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  PLBartTokenizer
       UpperCamelCase		=  None
       UpperCamelCase		=  False



       def   lowercase__       (							self    :						str					)				->		List[Any]:


             '''simple docstring'''




             super().setUp()

             # We have a SentencePiece fixture for testing
             UpperCAmelCase_    				=							PLBartTokenizer(_UpperCAmelCase	,       language_codes="base"	,       keep_accents=_UpperCAmelCase					)
             tokenizer.save_pretrained(self.tmpdirname					)



       def   lowercase__       (							self    :						Optional[int]					)				->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							PLBartTokenizer(_UpperCAmelCase	,       language_codes="base"	,       keep_accents=_UpperCAmelCase					)

             UpperCAmelCase_    				=							tokenizer.tokenize("This is a test"					)
             self.assertListEqual(_UpperCAmelCase	,       ["▁This", "▁is", "▁a", "▁t", "est"]					)

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)	,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,       )

             UpperCAmelCase_    				=							tokenizer.tokenize("I was born in 92000, and this is falsé."					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "9",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "é",
                     ".",
                 ]	,       )
             UpperCAmelCase_    				=							tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]	,       )

             UpperCAmelCase_    				=							tokenizer.convert_ids_to_tokens(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "<unk>",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "<unk>",
                     ".",
                 ]	,       )

             UpperCAmelCase_    				=							tokenizer.vocab_size
             UpperCAmelCase_    				=							[tokenizer.convert_ids_to_tokens(_UpperCAmelCase					) for x in range(end - 4	,       _UpperCAmelCase					)]

             self.assertListEqual(_UpperCAmelCase	,       ["__java__", "__python__", "__en_XX__", "<mask>"]					)

             UpperCAmelCase_    				=							"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
             UpperCAmelCase_    				=							tokenizer(_UpperCAmelCase					).input_ids
             self.assertEqual(
                 tokenizer.decode(_UpperCAmelCase	,       skip_special_tokens=_UpperCAmelCase	,       clean_up_tokenization_spaces=_UpperCAmelCase					)	,       _UpperCAmelCase	,       )







       def   lowercase__       (							self    :						Optional[Any]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							PLBartTokenizer(_UpperCAmelCase	,       language_codes="multi"	,       keep_accents=_UpperCAmelCase					)

             UpperCAmelCase_    				=							tokenizer.tokenize("This is a test"					)
             self.assertListEqual(_UpperCAmelCase	,       ["▁This", "▁is", "▁a", "▁t", "est"]					)

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)	,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,       )

             UpperCAmelCase_    				=							tokenizer.tokenize("I was born in 92000, and this is falsé."					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "9",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "é",
                     ".",
                 ]	,       )
             UpperCAmelCase_    				=							tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 ]	,       )

             UpperCAmelCase_    				=							tokenizer.convert_ids_to_tokens(_UpperCAmelCase					)
             self.assertListEqual(
                 _UpperCAmelCase	,       [
                     SPIECE_UNDERLINE + "I",
                     SPIECE_UNDERLINE + "was",
                     SPIECE_UNDERLINE + "b",
                     "or",
                     "n",
                     SPIECE_UNDERLINE + "in",
                     SPIECE_UNDERLINE + "",
                     "<unk>",
                     "2",
                     "0",
                     "0",
                     "0",
                     ",",
                     SPIECE_UNDERLINE + "and",
                     SPIECE_UNDERLINE + "this",
                     SPIECE_UNDERLINE + "is",
                     SPIECE_UNDERLINE + "f",
                     "al",
                     "s",
                     "<unk>",
                     ".",
                 ]	,       )
             UpperCAmelCase_    				=							tokenizer.vocab_size
             UpperCAmelCase_    				=							[tokenizer.convert_ids_to_tokens(_UpperCAmelCase					) for x in range(end - 7	,       _UpperCAmelCase					)]

             self.assertListEqual(
                 _UpperCAmelCase	,       ["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"]					)
             UpperCAmelCase_    				=							"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
             UpperCAmelCase_    				=							tokenizer(_UpperCAmelCase					).input_ids
             self.assertEqual(
                 tokenizer.decode(_UpperCAmelCase	,       skip_special_tokens=_UpperCAmelCase	,       clean_up_tokenization_spaces=_UpperCAmelCase					)	,       _UpperCAmelCase	,       )



@require_torch
@require_sentencepiece
@require_tokenizers
class    lowercase__     (					unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  '''uclanlp/plbart-python-en_XX'''
       UpperCamelCase		=  [
           '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
           '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
       ]
       UpperCamelCase		=  [
           '''Returns the maximum value of a b c.''',
           '''Sums the values of a b c.''',
       ]
       UpperCamelCase		=  [
           1_34,
           54_52,
           3_34_60,
           3_34_41,
           3_34_63,
           3_34_65,
           3_34_63,
           3_34_49,
           9_88,
           20,
           3_34_56,
           19,
           3_34_56,
           7_71,
           39,
           42_58,
           8_89,
           33_18,
           3_34_41,
           3_34_63,
           3_34_65,
           3_34_63,
           3_34_49,
           24_71,
           2,
           PYTHON_CODE,
       ]



       @classmethod
       def   lowercase__       (							cls    :						int					)				->		Dict:


             '''simple docstring'''




             UpperCAmelCase_    				=							PLBartTokenizer.from_pretrained(
                 cls.checkpoint_name	,       language_codes="base"	,       src_lang="python"	,       tgt_lang="en_XX"					)
             UpperCAmelCase_    				=							1
             return cls



       def   lowercase__       (							self    :						List[Any]					)				->		Optional[Any]:


             '''simple docstring'''




             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]	,       50001					)
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]	,       50002					)
             self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]	,       50003					)



       def   lowercase__       (							self    :						int					)				->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
             self.assertListEqual(self.expected_src_tokens	,       _UpperCAmelCase					)



       def   lowercase__       (							self    :						List[str]					)				->		Union[str, Any]:


             '''simple docstring'''




             self.assertIn(_UpperCAmelCase	,       self.tokenizer.all_special_ids					)
             UpperCAmelCase_    				=							[EN_CODE, 9037, 33442, 57, 752, 153, 14, 56, 18, 9, 2]
             UpperCAmelCase_    				=							self.tokenizer.decode(_UpperCAmelCase	,       skip_special_tokens=_UpperCAmelCase					)
             UpperCAmelCase_    				=							self.tokenizer.decode(generated_ids[1:]	,       skip_special_tokens=_UpperCAmelCase					)
             self.assertEqual(_UpperCAmelCase	,       _UpperCAmelCase					)
             self.assertNotIn(self.tokenizer.eos_token	,       _UpperCAmelCase					)



       def   lowercase__       (							self    :						int					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 20]
             self.assertIsInstance(src_text[0]	,       _UpperCAmelCase					)
             UpperCAmelCase_    				=							10
             UpperCAmelCase_    				=							self.tokenizer(_UpperCAmelCase	,       max_length=_UpperCAmelCase	,       truncation=_UpperCAmelCase					).input_ids[0]
             self.assertEqual(ids[-2]	,       2					)
             self.assertEqual(ids[-1]	,       _UpperCAmelCase					)
             self.assertEqual(len(_UpperCAmelCase					)	,       _UpperCAmelCase					)



       def   lowercase__       (							self    :						Optional[Any]					)				->		Optional[int]:


             '''simple docstring'''




             self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"]					)	,       [50004, 50001]					)



       def   lowercase__       (							self    :						Optional[Any]					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							tempfile.mkdtemp()
             UpperCAmelCase_    				=							self.tokenizer.fairseq_tokens_to_ids
             self.tokenizer.save_pretrained(_UpperCAmelCase					)
             UpperCAmelCase_    				=							PLBartTokenizer.from_pretrained(_UpperCAmelCase					)
             self.assertDictEqual(new_tok.fairseq_tokens_to_ids	,       _UpperCAmelCase					)



       @require_torch
       def   lowercase__       (							self    :						Optional[int]					)				->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer(self.src_text	,       text_target=self.tgt_text	,       padding=_UpperCAmelCase	,       return_tensors="pt"					)
             UpperCAmelCase_    				=							shift_tokens_right(batch["labels"]	,       self.tokenizer.pad_token_id					)

             # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
             self.assertEqual(batch.input_ids[1][-2:].tolist()	,       [2, PYTHON_CODE]					)
             self.assertEqual(batch.decoder_input_ids[1][0]	,       _UpperCAmelCase					)
             self.assertEqual(batch.decoder_input_ids[1][-1]	,       2					)
             self.assertEqual(batch.labels[1][-2:].tolist()	,       [2, EN_CODE]					)



       @require_torch
       def   lowercase__       (							self    :						int					)				->		str:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer(
                 self.src_text	,       text_target=self.tgt_text	,       padding=_UpperCAmelCase	,       truncation=_UpperCAmelCase	,       max_length=len(self.expected_src_tokens					)	,       return_tensors="pt"	,       )
             UpperCAmelCase_    				=							shift_tokens_right(batch["labels"]	,       self.tokenizer.pad_token_id					)

             self.assertIsInstance(_UpperCAmelCase	,       _UpperCAmelCase					)

             self.assertEqual((2, 26)	,       batch.input_ids.shape					)
             self.assertEqual((2, 26)	,       batch.attention_mask.shape					)
             UpperCAmelCase_    				=							batch.input_ids.tolist()[0]
             self.assertListEqual(self.expected_src_tokens	,       _UpperCAmelCase					)
             self.assertEqual(2	,       batch.decoder_input_ids[0, -1]					)  # EOS
             # Test that special tokens are reset
             self.assertEqual(self.tokenizer.prefix_tokens	,       []					)
             self.assertEqual(self.tokenizer.suffix_tokens	,       [self.tokenizer.eos_token_id, PYTHON_CODE]					)



       def   lowercase__       (							self    :						List[Any]					)				->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer(self.src_text	,       padding=_UpperCAmelCase	,       truncation=_UpperCAmelCase	,       max_length=3	,       return_tensors="pt"					)
             UpperCAmelCase_    				=							self.tokenizer(
                 text_target=self.tgt_text	,       padding=_UpperCAmelCase	,       truncation=_UpperCAmelCase	,       max_length=10	,       return_tensors="pt"					)
             UpperCAmelCase_    				=							targets["input_ids"]
             UpperCAmelCase_    				=							shift_tokens_right(_UpperCAmelCase	,       self.tokenizer.pad_token_id					)

             self.assertEqual(batch.input_ids.shape[1]	,       3					)
             self.assertEqual(batch.decoder_input_ids.shape[1]	,       10					)







       @require_torch
       def   lowercase__       (							self    :						Tuple					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer._build_translation_inputs(
                 "A test"	,       return_tensors="pt"	,       src_lang="en_XX"	,       tgt_lang="java"					)

             self.assertEqual(
                 nested_simplify(_UpperCAmelCase					)	,       {
                     # A, test, EOS, en_XX
                     "input_ids": [[150, 242, 2, 50003]],
                     "attention_mask": [[1, 1, 1, 1]],
                     # java
                     "forced_bos_token_id": 50001,
                 }	,       )




