



"""simple docstring"""





import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      _lowercase (     _snake_case		,  unittest.TestCase						):
      _a   :     Tuple								=   LEDTokenizer
      _a   :     Optional[int]								=   LEDTokenizerFast
      _a   :     Optional[Any]								=   True


      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            super().setUp()
            __snake_case      :    Optional[int]  			=[
                '''l''',
                '''o''',
                '''w''',
                '''e''',
                '''r''',
                '''s''',
                '''t''',
                '''i''',
                '''d''',
                '''n''',
                '''\u0120''',
                '''\u0120l''',
                '''\u0120n''',
                '''\u0120lo''',
                '''\u0120low''',
                '''er''',
                '''\u0120lowest''',
                '''\u0120newer''',
                '''\u0120wider''',
                '''<unk>''',
            ]
            __snake_case      :    int  			=dict(zip(lowerCAmelCase__							,   range(len(lowerCAmelCase__						)						)						)						)
            __snake_case      :    Dict  			=['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
            __snake_case      :    Tuple  			={'''unk_token''': '''<unk>'''}

            __snake_case      :    str  			=os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['''vocab_file''']						)
            __snake_case      :    Dict  			=os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['''merges_file''']						)
            with open(self.vocab_file							,   '''w'''							,   encoding='''utf-8'''						) as fp:
                  fp.write(json.dumps(lowerCAmelCase__						) + '''\n'''						)
            with open(self.merges_file							,   '''w'''							,   encoding='''utf-8'''						) as fp:
                  fp.write('''\n'''.join(lowerCAmelCase__						)						)


      def _UpperCamelCase						(    self				:	Union[str, Any]							,   **a				:	Optional[int]						):





            """simple docstring"""







            kwargs.update(self.special_tokens_map						)
            return self.tokenizer_class.from_pretrained(self.tmpdirname							,   **lowerCAmelCase__						)


      def _UpperCamelCase						(    self				:	str							,   **a				:	str						):





            """simple docstring"""







            kwargs.update(self.special_tokens_map						)
            return self.rust_tokenizer_class.from_pretrained(self.tmpdirname							,   **lowerCAmelCase__						)


      def _UpperCamelCase						(    self				:	List[str]							,   a				:	List[Any]						):





            """simple docstring"""







            return "lower newer", "lower newer"


      @cached_property
      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            return LEDTokenizer.from_pretrained('''allenai/led-base-16384'''						)


      @cached_property
      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            return LEDTokenizerFast.from_pretrained('''allenai/led-base-16384'''						)


      @require_torch
      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            __snake_case      :    Optional[int]  			=['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
            __snake_case      :    int  			=[0, 2_5_0, 2_5_1, 1_7_8_1_8, 1_3, 3_9_1_8_6, 1_9_3_8, 4, 2]

            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  __snake_case      :    int  			=tokenizer(lowerCAmelCase__							,   max_length=len(lowerCAmelCase__						)							,   padding=lowerCAmelCase__							,   return_tensors='''pt'''						)
                  self.assertIsInstance(lowerCAmelCase__							,   lowerCAmelCase__						)

                  self.assertEqual((2, 9)							,   batch.input_ids.shape						)
                  self.assertEqual((2, 9)							,   batch.attention_mask.shape						)
                  __snake_case      :    Dict  			=batch.input_ids.tolist()[0]
                  self.assertListEqual(lowerCAmelCase__							,   lowerCAmelCase__						)


      @require_torch
      def _UpperCamelCase						(    self				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case      :    Optional[int]  			=['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  __snake_case      :    Tuple  			=tokenizer(lowerCAmelCase__							,   padding=lowerCAmelCase__							,   return_tensors='''pt'''						)
                  self.assertIn('''input_ids'''							,   lowerCAmelCase__						)
                  self.assertIn('''attention_mask'''							,   lowerCAmelCase__						)
                  self.assertNotIn('''labels'''							,   lowerCAmelCase__						)
                  self.assertNotIn('''decoder_attention_mask'''							,   lowerCAmelCase__						)


      @require_torch
      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            __snake_case      :    Optional[int]  			=[
                '''Summary of the text.''',
                '''Another summary.''',
            ]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  __snake_case      :    Union[str, Any]  			=tokenizer(text_target=lowerCAmelCase__							,   max_length=3_2							,   padding='''max_length'''							,   return_tensors='''pt'''						)
                  self.assertEqual(3_2							,   targets['''input_ids'''].shape[1]						)


      @require_torch
      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  __snake_case      :    Dict  			=tokenizer(
                      ['''I am a small frog''' * 1_0_2_4, '''I am a small frog''']							,   padding=lowerCAmelCase__							,   truncation=lowerCAmelCase__							,   return_tensors='''pt'''						)
                  self.assertIsInstance(lowerCAmelCase__							,   lowerCAmelCase__						)
                  self.assertEqual(batch.input_ids.shape							,   (2, 5_1_2_2)						)


      @require_torch
      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    Dict  			=['''A long paragraph for summarization.''']
            __snake_case      :    Tuple  			=[
                '''Summary of the text.''',
            ]
            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  __snake_case      :    Tuple  			=tokenizer(lowerCAmelCase__							,   return_tensors='''pt'''						)
                  __snake_case      :    Tuple  			=tokenizer(text_target=lowerCAmelCase__							,   return_tensors='''pt'''						)
                  __snake_case      :    str  			=inputs['''input_ids''']
                  __snake_case      :    List[Any]  			=targets['''input_ids''']
                  self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()						)
                  self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()						)
                  self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()						)
                  self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()						)


      @require_torch
      def _UpperCamelCase						(    self				:	Optional[int]						):





            """simple docstring"""







            for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                  __snake_case      :    List[Any]  			=['''Summary of the text.''', '''Another summary.''']
                  __snake_case      :    List[Any]  			=[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

                  __snake_case      :    int  			=tokenizer(lowerCAmelCase__							,   padding=lowerCAmelCase__						)
                  __snake_case      :    str  			=[[0] * len(lowerCAmelCase__						) for x in encoded_output['''input_ids''']]
                  __snake_case      :    str  			=tokenizer.pad(lowerCAmelCase__						)
                  self.assertSequenceEqual(outputs['''global_attention_mask''']							,   lowerCAmelCase__						)


      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            pass







      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
                        __snake_case      :    Union[str, Any]  			=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__							,   **lowerCAmelCase__						)
                        __snake_case      :    Optional[int]  			=self.tokenizer_class.from_pretrained(lowerCAmelCase__							,   **lowerCAmelCase__						)
                        __snake_case      :    Optional[Any]  			='''A, <mask> AllenNLP sentence.'''
                        __snake_case      :    Tuple  			=tokenizer_r.encode_plus(lowerCAmelCase__							,   add_special_tokens=lowerCAmelCase__							,   return_token_type_ids=lowerCAmelCase__						)
                        __snake_case      :    Any  			=tokenizer_p.encode_plus(lowerCAmelCase__							,   add_special_tokens=lowerCAmelCase__							,   return_token_type_ids=lowerCAmelCase__						)
                        self.assertEqual(sum(tokens_r['''token_type_ids''']						)							,   sum(tokens_p['''token_type_ids''']						)						)
                        self.assertEqual(
                            sum(tokens_r['''attention_mask''']						) / len(tokens_r['''attention_mask''']						)							,   sum(tokens_p['''attention_mask''']						) / len(tokens_p['''attention_mask''']						)							,   )

                        __snake_case      :    Tuple  			=tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']						)
                        __snake_case      :    Dict  			=tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']						)
                        self.assertSequenceEqual(tokens_p['''input_ids''']							,   [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]						)
                        self.assertSequenceEqual(tokens_r['''input_ids''']							,   [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]						)

                        self.assertSequenceEqual(
                            lowerCAmelCase__							,   ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']						)
                        self.assertSequenceEqual(
                            lowerCAmelCase__							,   ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']						)










"""simple docstring"""





import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand



def     __lowercase					(    a :				Dict )					->					str:
      return EnvironmentCommand()



def     __lowercase					(    a :				Optional[int] )					->					List[Any]:
      return EnvironmentCommand(args.accelerate_config_file )


class      _lowercase (     lowerCAmelCase						):


      @staticmethod
      def _UpperCamelCase						(    a				:	ArgumentParser						):





            """simple docstring"""







            __snake_case      :    Optional[int]  			=parser.add_parser('''env'''						)
            download_parser.set_defaults(func=a						)
            download_parser.add_argument(
                '''--accelerate-config_file'''							,   default=a							,   help='''The accelerate config file to use for the default values in the launching script.'''							,   )
            download_parser.set_defaults(func=a						)


      def __init__(    self				:	Any							,   a				:	str							,   *a				:	Any						):





            """simple docstring"""







            __snake_case      :    int  			=accelerate_config_file


      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            __snake_case      :    List[str]  			='''not installed'''
            if is_safetensors_available():
                  import safetensors

                  __snake_case      :    int  			=safetensors.__version__
            elif importlib.util.find_spec('''safetensors'''						) is not None:
                  import safetensors

                  __snake_case      :    List[Any]  			=f'''{safetensors.__version__} but is ignored because of PyTorch version too old.'''

            __snake_case      :    Dict  			='''not installed'''
            __snake_case      :    Optional[Any]  			='''not found'''
            if is_accelerate_available():
                  import accelerate
                  from accelerate.commands.config import default_config_file, load_config_from_file

                  __snake_case      :    Union[str, Any]  			=accelerate.__version__
                  # Get the default from the config file.
                  if self._accelerate_config_file is not None or os.path.isfile(a						):
                        __snake_case      :    int  			=load_config_from_file(self._accelerate_config_file						).to_dict()

                  __snake_case      :    Optional[int]  			=(
                      '''\n'''.join([f'''\t- {prop}: {val}''' for prop, val in accelerate_config.items()]						)
                      if isinstance(a							,   a						)
                      else f'''\t{accelerate_config}'''
                  )

            __snake_case      :    int  			='''not installed'''
            __snake_case      :    Any  			='''NA'''
            if is_torch_available():
                  import torch

                  __snake_case      :    Tuple  			=torch.__version__
                  __snake_case      :    Optional[Any]  			=torch.cuda.is_available()

            __snake_case      :    Optional[Any]  			='''not installed'''
            __snake_case      :    Any  			='''NA'''
            if is_tf_available():
                  import tensorflow as tf

                  __snake_case      :    Optional[Any]  			=tf.__version__
                  try:
                        # deprecated in v2.1
                        __snake_case      :    str  			=tf.test.is_gpu_available()
                  except AttributeError:
                        # returns list of devices, convert to bool
                        __snake_case      :    List[Any]  			=bool(tf.config.list_physical_devices('''GPU'''						)						)

            __snake_case      :    Optional[int]  			='''not installed'''
            __snake_case      :    Dict  			='''not installed'''
            __snake_case      :    List[str]  			='''not installed'''
            __snake_case      :    Optional[int]  			='''NA'''
            if is_flax_available():
                  import flax
                  import jax
                  import jaxlib

                  __snake_case      :    Tuple  			=flax.__version__
                  __snake_case      :    Optional[Any]  			=jax.__version__
                  __snake_case      :    str  			=jaxlib.__version__
                  __snake_case      :    Tuple  			=jax.lib.xla_bridge.get_backend().platform

            __snake_case      :    List[str]  			={
                '''`transformers` version''': version,
                '''Platform''': platform.platform(),
                '''Python version''': platform.python_version(),
                '''Huggingface_hub version''': huggingface_hub.__version__,
                '''Safetensors version''': f'''{safetensors_version}''',
                '''Accelerate version''': f'''{accelerate_version}''',
                '''Accelerate config''': f'''{accelerate_config_str}''',
                '''PyTorch version (GPU?)''': f'''{pt_version} ({pt_cuda_available})''',
                '''Tensorflow version (GPU?)''': f'''{tf_version} ({tf_cuda_available})''',
                '''Flax version (CPU?/GPU?/TPU?)''': f'''{flax_version} ({jax_backend})''',
                '''Jax version''': f'''{jax_version}''',
                '''JaxLib version''': f'''{jaxlib_version}''',
                '''Using GPU in script?''': '''<fill in>''',
                '''Using distributed or parallel set-up in script?''': '''<fill in>''',
            }

            print('''\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n'''						)
            print(self.format_dict(a						)						)

            return info







      @staticmethod
      def _UpperCamelCase						(    a				:	List[str]						):





            """simple docstring"""







            return "\n".join([f'''- {prop}: {val}''' for prop, val in d.items()]						) + "\n"


