





'''simple docstring'''

import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase					:		List[str]		=							get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
   from transformers.models.plbart.modeling_plbart import shift_tokens_right

UpperCAmelCase					:		Any		=							5_0_0_0_3
UpperCAmelCase					:		Optional[Any]		=							5_0_0_0_2





@require_sentencepiece
@require_tokenizers
class lowerCamelCase						(a__						,      unittest.TestCase					):
  _lowercase  :	Any       					=				PLBartTokenizer
  _lowercase  :	Optional[Any]       					=				None
  _lowercase  :	List[str]       					=				False







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        super().setUp()

        # We have a SentencePiece fixture for testing
        _snake_case		:	Dict					     =     PLBartTokenizer(lowercase__     ,     language_codes='''base'''     ,     keep_accents=lowercase__		)
        tokenizer.save_pretrained(self.tmpdirname		)







  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     PLBartTokenizer(lowercase__     ,     language_codes='''base'''     ,     keep_accents=lowercase__		)

        _snake_case		:	int					     =     tokenizer.tokenize('''This is a test'''		)
        self.assertListEqual(lowercase__     ,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(lowercase__		)     ,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,     )

        _snake_case		:	Tuple					     =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
        self.assertListEqual(
            lowercase__     ,     [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ]     ,     )
        _snake_case		:	Optional[int]					     =     tokenizer.convert_tokens_to_ids(lowercase__		)
        self.assertListEqual(
            lowercase__     ,     [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ]     ,     )

        _snake_case		:	Optional[Any]					     =     tokenizer.convert_ids_to_tokens(lowercase__		)
        self.assertListEqual(
            lowercase__     ,     [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ]     ,     )

        _snake_case		:	Any					     =     tokenizer.vocab_size
        _snake_case		:	int					     =     [tokenizer.convert_ids_to_tokens(lowercase__		) for x in range(end - 4     ,     lowercase__		)]

        self.assertListEqual(lowercase__     ,     ['''__java__''', '''__python__''', '''__en_XX__''', '''<mask>''']		)

        _snake_case		:	Tuple					     =     '''java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'''
        _snake_case		:	Dict					     =     tokenizer(lowercase__		).input_ids
        self.assertEqual(
            tokenizer.decode(lowercase__     ,     skip_special_tokens=lowercase__     ,     clean_up_tokenization_spaces=lowercase__		)     ,     lowercase__     ,     )







  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	Union[str, Any]					     =     PLBartTokenizer(lowercase__     ,     language_codes='''multi'''     ,     keep_accents=lowercase__		)

        _snake_case		:	Dict					     =     tokenizer.tokenize('''This is a test'''		)
        self.assertListEqual(lowercase__     ,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(lowercase__		)     ,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,     )

        _snake_case		:	Dict					     =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
        self.assertListEqual(
            lowercase__     ,     [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ]     ,     )
        _snake_case		:	List[Any]					     =     tokenizer.convert_tokens_to_ids(lowercase__		)
        self.assertListEqual(
            lowercase__     ,     [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ]     ,     )

        _snake_case		:	Any					     =     tokenizer.convert_ids_to_tokens(lowercase__		)
        self.assertListEqual(
            lowercase__     ,     [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ]     ,     )
        _snake_case		:	List[Any]					     =     tokenizer.vocab_size
        _snake_case		:	Any					     =     [tokenizer.convert_ids_to_tokens(lowercase__		) for x in range(end - 7     ,     lowercase__		)]

        self.assertListEqual(
            lowercase__     ,     ['''__java__''', '''__python__''', '''__en_XX__''', '''__javascript__''', '''__php__''', '''__ruby__''', '''__go__''']		)
        _snake_case		:	Optional[int]					     =     '''java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'''
        _snake_case		:	str					     =     tokenizer(lowercase__		).input_ids
        self.assertEqual(
            tokenizer.decode(lowercase__     ,     skip_special_tokens=lowercase__     ,     clean_up_tokenization_spaces=lowercase__		)     ,     lowercase__     ,     )





@require_torch
@require_sentencepiece
@require_tokenizers
class lowerCamelCase						(unittest.TestCase					):
  _lowercase  :	Union[str, Any]       					=				"""uclanlp/plbart-python-en_XX"""
  _lowercase  :	Optional[int]       					=				[
      """def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])""",
      """def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""",
  ]
  _lowercase  :	List[Any]       					=				[
      """Returns the maximum value of a b c.""",
      """Sums the values of a b c.""",
  ]
  _lowercase  :	List[Any]       					=				[
      134,
      5_452,
      33_460,
      33_441,
      33_463,
      33_465,
      33_463,
      33_449,
      988,
      20,
      33_456,
      19,
      33_456,
      771,
      39,
      4_258,
      889,
      3_318,
      33_441,
      33_463,
      33_465,
      33_463,
      33_449,
      2_471,
      2,
      PYTHON_CODE,
  ]







  @classmethod
  def 				UpperCAmelCase_				(		cls		)		->						List[str]:






        """simple docstring"""





        _snake_case		:	PLBartTokenizer					     =     PLBartTokenizer.from_pretrained(
            cls.checkpoint_name     ,     language_codes='''base'''     ,     src_lang='''python'''     ,     tgt_lang='''en_XX'''		)
        _snake_case		:	Optional[Any]					     =     1
        return cls







  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''__java__''']     ,     50_001		)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''__python__''']     ,     50_002		)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''__en_XX__''']     ,     50_003		)







  def 				UpperCAmelCase_				(		self		)		->						int:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
        self.assertListEqual(self.expected_src_tokens     ,     lowercase__		)







  def 				UpperCAmelCase_				(		self		)		->						Optional[int]:






        """simple docstring"""





        self.assertIn(lowercase__     ,     self.tokenizer.all_special_ids		)
        _snake_case		:	Optional[Any]					     =     [EN_CODE, 9_037, 33_442, 57, 752, 153, 14, 56, 18, 9, 2]
        _snake_case		:	Any					     =     self.tokenizer.decode(lowercase__     ,     skip_special_tokens=lowercase__		)
        _snake_case		:	Tuple					     =     self.tokenizer.decode(generated_ids[1:]     ,     skip_special_tokens=lowercase__		)
        self.assertEqual(lowercase__     ,     lowercase__		)
        self.assertNotIn(self.tokenizer.eos_token     ,     lowercase__		)







  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        _snake_case		:	Tuple					     =     ['''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''' * 20]
        self.assertIsInstance(src_text[0]     ,     lowercase__		)
        _snake_case		:	int					     =     10
        _snake_case		:	int					     =     self.tokenizer(lowercase__     ,     max_length=lowercase__     ,     truncation=lowercase__		).input_ids[0]
        self.assertEqual(ids[-2]     ,     2		)
        self.assertEqual(ids[-1]     ,     lowercase__		)
        self.assertEqual(len(lowercase__		)     ,     lowercase__		)







  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''__java__''']		)     ,     [50_004, 50_001]		)







  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	Optional[int]					     =     tempfile.mkdtemp()
        _snake_case		:	Optional[Any]					     =     self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(lowercase__		)
        _snake_case		:	List[Any]					     =     PLBartTokenizer.from_pretrained(lowercase__		)
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids     ,     lowercase__		)







  @require_torch
  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	List[str]					     =     self.tokenizer(self.src_text     ,     text_target=self.tgt_text     ,     padding=lowercase__     ,     return_tensors='''pt'''		)
        _snake_case		:	Any					     =     shift_tokens_right(batch['''labels''']     ,     self.tokenizer.pad_token_id		)

        # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
        self.assertEqual(batch.input_ids[1][-2:].tolist()     ,     [2, PYTHON_CODE]		)
        self.assertEqual(batch.decoder_input_ids[1][0]     ,     lowercase__		)
        self.assertEqual(batch.decoder_input_ids[1][-1]     ,     2		)
        self.assertEqual(batch.labels[1][-2:].tolist()     ,     [2, EN_CODE]		)







  @require_torch
  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        _snake_case		:	List[str]					     =     self.tokenizer(
            self.src_text     ,     text_target=self.tgt_text     ,     padding=lowercase__     ,     truncation=lowercase__     ,     max_length=len(self.expected_src_tokens		)     ,     return_tensors='''pt'''     ,     )
        _snake_case		:	List[str]					     =     shift_tokens_right(batch['''labels''']     ,     self.tokenizer.pad_token_id		)

        self.assertIsInstance(lowercase__     ,     lowercase__		)

        self.assertEqual((2, 26)     ,     batch.input_ids.shape		)
        self.assertEqual((2, 26)     ,     batch.attention_mask.shape		)
        _snake_case		:	List[str]					     =     batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens     ,     lowercase__		)
        self.assertEqual(2     ,     batch.decoder_input_ids[0, -1]		)  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens     ,     []		)
        self.assertEqual(self.tokenizer.suffix_tokens     ,     [self.tokenizer.eos_token_id, PYTHON_CODE]		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	Dict					     =     self.tokenizer(self.src_text     ,     padding=lowercase__     ,     truncation=lowercase__     ,     max_length=3     ,     return_tensors='''pt'''		)
        _snake_case		:	Tuple					     =     self.tokenizer(
            text_target=self.tgt_text     ,     padding=lowercase__     ,     truncation=lowercase__     ,     max_length=10     ,     return_tensors='''pt'''		)
        _snake_case		:	Optional[int]					     =     targets['''input_ids''']
        _snake_case		:	Optional[Any]					     =     shift_tokens_right(lowercase__     ,     self.tokenizer.pad_token_id		)

        self.assertEqual(batch.input_ids.shape[1]     ,     3		)
        self.assertEqual(batch.decoder_input_ids.shape[1]     ,     10		)







  @require_torch
  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        _snake_case		:	int					     =     self.tokenizer._build_translation_inputs(
            '''A test'''     ,     return_tensors='''pt'''     ,     src_lang='''en_XX'''     ,     tgt_lang='''java'''		)

        self.assertEqual(
            nested_simplify(lowercase__		)     ,     {
                # A, test, EOS, en_XX
                '''input_ids''': [[150, 242, 2, 50_003]],
                '''attention_mask''': [[1, 1, 1, 1]],
                # java
                '''forced_bos_token_id''': 50_001,
            }     ,     )









'''simple docstring'''

from argparse import ArgumentParser

from .add_new_model import AddNewModelCommand
from .add_new_model_like import AddNewModelLikeCommand
from .convert import ConvertCommand
from .download import DownloadCommand
from .env import EnvironmentCommand
from .lfs import LfsCommands
from .pt_to_tf import PTtoTFCommand
from .run import RunCommand
from .serving import ServeCommand
from .user import UserCommands




def    _a    (							):





      """simple docstring"""



      _snake_case		:	List[Any]					     =     ArgumentParser('''Transformers CLI tool'''			,      usage='''transformers-cli <command> [<args>]'''	)
      _snake_case		:	List[str]					     =     parser.add_subparsers(help='''transformers-cli command helpers'''	)

      # Register commands
      ConvertCommand.register_subcommand(lowerCAmelCase_	)
      DownloadCommand.register_subcommand(lowerCAmelCase_	)
      EnvironmentCommand.register_subcommand(lowerCAmelCase_	)
      RunCommand.register_subcommand(lowerCAmelCase_	)
      ServeCommand.register_subcommand(lowerCAmelCase_	)
      UserCommands.register_subcommand(lowerCAmelCase_	)
      AddNewModelCommand.register_subcommand(lowerCAmelCase_	)
      AddNewModelLikeCommand.register_subcommand(lowerCAmelCase_	)
      LfsCommands.register_subcommand(lowerCAmelCase_	)
      PTtoTFCommand.register_subcommand(lowerCAmelCase_	)

      # Let's go
      _snake_case		:	str					     =     parser.parse_args()

      if not hasattr(lowerCAmelCase_			,      '''func'''	):
            parser.print_help()
            exit(1	)

      # Run
      _snake_case		:	Union[str, Any]					     =     args.func(lowerCAmelCase_	)
      service.run()


if __name__ == "__main__":
   main()


