


'''simple docstring'''


import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY



@is_pipeline_test
@require_torch_or_tf
class 	__lowerCAmelCase		(							unittest.TestCase    ):


  '''simple docstring'''







  a_  					=							MODEL_FOR_CAUSAL_LM_MAPPING
  a_  					=							TF_MODEL_FOR_CAUSAL_LM_MAPPING






  @require_torch
  def        _a							(							self :       Optional[Any] ):

        '''simple docstring'''




        A_						:  str          =				pipeline(task="""text-generation"""	,model="""sshleifer/tiny-ctrl"""	,framework="""pt""" )
        # Using `do_sample=False` to force deterministic output
        A_						:  Optional[int]          =				text_generator("""This is a test"""	,do_sample=__lowerCamelCase )
        self.assertEqual(
            __lowerCamelCase	,[
                {
                    """generated_text""": (
                        """This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."""
                        """ oscope. FiliFili@@"""
                    )
                }
            ]	,)

        A_						:  Dict          =				text_generator(["""This is a test""", """This is a second test"""] )
        self.assertEqual(
            __lowerCamelCase	,[
                [
                    {
                        """generated_text""": (
                            """This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."""
                            """ oscope. FiliFili@@"""
                        )
                    }
                ],
                [
                    {
                        """generated_text""": (
                            """This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy"""
                            """ oscope. oscope. FiliFili@@"""
                        )
                    }
                ],
            ]	,)

        A_						:  Optional[int]          =				text_generator("""This is a test"""	,do_sample=__lowerCamelCase	,num_return_sequences=2	,return_tensors=__lowerCamelCase )
        self.assertEqual(
            __lowerCamelCase	,[
                {"""generated_token_ids""": ANY(__lowerCamelCase )},
                {"""generated_token_ids""": ANY(__lowerCamelCase )},
            ]	,)
        A_						:  Union[str, Any]          =				text_generator.model.config.eos_token_id
        A_						:  str          =				'''<pad>'''
        A_						:  Optional[int]          =				text_generator(
            ["""This is a test""", """This is a second test"""]	,do_sample=__lowerCamelCase	,num_return_sequences=2	,batch_size=2	,return_tensors=__lowerCamelCase	,)
        self.assertEqual(
            __lowerCamelCase	,[
                [
                    {"""generated_token_ids""": ANY(__lowerCamelCase )},
                    {"""generated_token_ids""": ANY(__lowerCamelCase )},
                ],
                [
                    {"""generated_token_ids""": ANY(__lowerCamelCase )},
                    {"""generated_token_ids""": ANY(__lowerCamelCase )},
                ],
            ]	,)






  @require_tf
  def        _a							(							self :       List[Any] ):

        '''simple docstring'''




        A_						:  int          =				pipeline(task="""text-generation"""	,model="""sshleifer/tiny-ctrl"""	,framework="""tf""" )

        # Using `do_sample=False` to force deterministic output
        A_						:  str          =				text_generator("""This is a test"""	,do_sample=__lowerCamelCase )
        self.assertEqual(
            __lowerCamelCase	,[
                {
                    """generated_text""": (
                        """This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"""
                        """ please,"""
                    )
                }
            ]	,)

        A_						:  List[str]          =				text_generator(["""This is a test""", """This is a second test"""]	,do_sample=__lowerCamelCase )
        self.assertEqual(
            __lowerCamelCase	,[
                [
                    {
                        """generated_text""": (
                            """This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"""
                            """ please,"""
                        )
                    }
                ],
                [
                    {
                        """generated_text""": (
                            """This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes"""
                            """ Cannes 閲閲Cannes Cannes Cannes 攵 please,"""
                        )
                    }
                ],
            ]	,)






  def        _a							(							self :       List[Any]	,_a :       List[str]	,_a :       Tuple	,_a :       str ):

        '''simple docstring'''




        A_						:  int          =				TextGenerationPipeline(model=__lowerCamelCase	,tokenizer=__lowerCamelCase )
        return text_generator, ["This is a test", "Another test"]






  def        _a							(							self :       Dict ):

        '''simple docstring'''




        A_						:  Tuple          =				'''Hello I believe in'''
        A_						:  str          =				pipeline("""text-generation"""	,model="""hf-internal-testing/tiny-random-gpt2""" )
        A_						:  Dict          =				text_generator(__lowerCamelCase )
        self.assertEqual(
            __lowerCamelCase	,[{"""generated_text""": """Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe"""}]	,)

        A_						:  Dict          =				text_generator(__lowerCamelCase	,stop_sequence=""" fe""" )
        self.assertEqual(__lowerCamelCase	,[{"""generated_text""": """Hello I believe in fe"""}] )






  def        _a							(							self :       Optional[Any]	,_a :       Any	,_a :       List[Any] ):

        '''simple docstring'''




        A_						:  Optional[Any]          =				text_generator.model
        A_						:  Union[str, Any]          =				text_generator.tokenizer

        A_						:  Optional[int]          =				text_generator("""This is a test""" )
        self.assertEqual(__lowerCamelCase	,[{"""generated_text""": ANY(__lowerCamelCase )}] )
        self.assertTrue(outputs[0]["""generated_text"""].startswith("""This is a test""" ) )

        A_						:  List[Any]          =				text_generator("""This is a test"""	,return_full_text=__lowerCamelCase )
        self.assertEqual(__lowerCamelCase	,[{"""generated_text""": ANY(__lowerCamelCase )}] )
        self.assertNotIn("""This is a test"""	,outputs[0]["""generated_text"""] )

        A_						:  str          =				pipeline(task="""text-generation"""	,model=__lowerCamelCase	,tokenizer=__lowerCamelCase	,return_full_text=__lowerCamelCase )
        A_						:  Optional[Any]          =				text_generator("""This is a test""" )
        self.assertEqual(__lowerCamelCase	,[{"""generated_text""": ANY(__lowerCamelCase )}] )
        self.assertNotIn("""This is a test"""	,outputs[0]["""generated_text"""] )

        A_						:  List[Any]          =				text_generator("""This is a test"""	,return_full_text=__lowerCamelCase )
        self.assertEqual(__lowerCamelCase	,[{"""generated_text""": ANY(__lowerCamelCase )}] )
        self.assertTrue(outputs[0]["""generated_text"""].startswith("""This is a test""" ) )

        A_						:  List[str]          =				text_generator(["""This is great !""", """Something else"""]	,num_return_sequences=2	,do_sample=__lowerCamelCase )
        self.assertEqual(
            __lowerCamelCase	,[
                [{"""generated_text""": ANY(__lowerCamelCase )}, {"""generated_text""": ANY(__lowerCamelCase )}],
                [{"""generated_text""": ANY(__lowerCamelCase )}, {"""generated_text""": ANY(__lowerCamelCase )}],
            ]	,)

        if text_generator.tokenizer.pad_token is not None:
              A_						:  List[str]          =				text_generator(
                  ["""This is great !""", """Something else"""]	,num_return_sequences=2	,batch_size=2	,do_sample=__lowerCamelCase )
              self.assertEqual(
                  __lowerCamelCase	,[
                      [{"""generated_text""": ANY(__lowerCamelCase )}, {"""generated_text""": ANY(__lowerCamelCase )}],
                      [{"""generated_text""": ANY(__lowerCamelCase )}, {"""generated_text""": ANY(__lowerCamelCase )}],
                  ]	,)

        with self.assertRaises(__lowerCamelCase ):
              A_						:  List[str]          =				text_generator("""test"""	,return_full_text=__lowerCamelCase	,return_text=__lowerCamelCase )
        with self.assertRaises(__lowerCamelCase ):
              A_						:  List[Any]          =				text_generator("""test"""	,return_full_text=__lowerCamelCase	,return_tensors=__lowerCamelCase )
        with self.assertRaises(__lowerCamelCase ):
              A_						:  Union[str, Any]          =				text_generator("""test"""	,return_text=__lowerCamelCase	,return_tensors=__lowerCamelCase )

        # Empty prompt is slighly special
        # it requires BOS token to exist.
        # Special case for Pegasus which will always append EOS so will
        # work even without BOS.
        if (
            text_generator.tokenizer.bos_token_id is not None
            or "Pegasus" in tokenizer.__class__.__name__
            or "Git" in model.__class__.__name__
        ):
              A_						:  int          =				text_generator("""""" )
              self.assertEqual(__lowerCamelCase	,[{"""generated_text""": ANY(__lowerCamelCase )}] )
        else:
              with self.assertRaises((ValueError, AssertionError) ):
                    A_						:  List[Any]          =				text_generator("""""" )

        if text_generator.framework == "tf":
              # TF generation does not support max_new_tokens, and it's impossible
              # to control long generation with only max_length without
              # fancy calculation, dismissing tests for now.
              return
        # We don't care about infinite range models.
        # They already work.
        # Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
        A_						:  Tuple          =				['''RwkvForCausalLM''', '''XGLMForCausalLM''', '''GPTNeoXForCausalLM''']
        if (
            tokenizer.model_max_length < 10000
            and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
        ):
              # Handling of large generations
              with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError) ):
                    text_generator("""This is a test""" * 500	,max_new_tokens=20 )

              A_						:  List[str]          =				text_generator("""This is a test""" * 500	,handle_long_generation="""hole"""	,max_new_tokens=20 )
              # Hole strategy cannot work
              with self.assertRaises(__lowerCamelCase ):
                    text_generator(
                        """This is a test""" * 500	,handle_long_generation="""hole"""	,max_new_tokens=tokenizer.model_max_length + 10	,)






  @require_torch
  @require_accelerate
  @require_torch_gpu
  def        _a							(							self :       List[str] ):

        '''simple docstring'''




        import torch

        # Classic `model_kwargs`
        A_						:  List[str]          =				pipeline(
            model="""hf-internal-testing/tiny-random-bloom"""	,model_kwargs={"""device_map""": """auto""", """torch_dtype""": torch.bfloataa}	,)
        self.assertEqual(pipe.model.device	,torch.device(0 ) )
        self.assertEqual(pipe.model.lm_head.weight.dtype	,torch.bfloataa )
        A_						:  Any          =				pipe("""This is a test""" )
        self.assertEqual(
            __lowerCamelCase	,[
                {
                    """generated_text""": (
                        """This is a test test test test test test test test test test test test test test test test"""
                        """ test"""
                    )
                }
            ]	,)

        # Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
        A_						:  Union[str, Any]          =				pipeline(model="""hf-internal-testing/tiny-random-bloom"""	,device_map="""auto"""	,torch_dtype=torch.bfloataa )
        self.assertEqual(pipe.model.device	,torch.device(0 ) )
        self.assertEqual(pipe.model.lm_head.weight.dtype	,torch.bfloataa )
        A_						:  List[Any]          =				pipe("""This is a test""" )
        self.assertEqual(
            __lowerCamelCase	,[
                {
                    """generated_text""": (
                        """This is a test test test test test test test test test test test test test test test test"""
                        """ test"""
                    )
                }
            ]	,)

        # torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
        A_						:  Optional[Any]          =				pipeline(model="""hf-internal-testing/tiny-random-bloom"""	,device_map="""auto""" )
        self.assertEqual(pipe.model.device	,torch.device(0 ) )
        self.assertEqual(pipe.model.lm_head.weight.dtype	,torch.floataa )
        A_						:  Tuple          =				pipe("""This is a test""" )
        self.assertEqual(
            __lowerCamelCase	,[
                {
                    """generated_text""": (
                        """This is a test test test test test test test test test test test test test test test test"""
                        """ test"""
                    )
                }
            ]	,)






  @require_torch
  @require_torch_gpu
  def        _a							(							self :       Optional[Any] ):

        '''simple docstring'''




        import torch

        A_						:  Any          =				pipeline(model="""hf-internal-testing/tiny-random-bloom"""	,device=0	,torch_dtype=torch.floataa )
        pipe("""This is a test""" )






  @require_torch
  @require_accelerate
  @require_torch_gpu
  def        _a							(							self :       Optional[Any] ):

        '''simple docstring'''




        import torch

        A_						:  Any          =				pipeline(model="""hf-internal-testing/tiny-random-bloom"""	,device_map="""auto"""	,torch_dtype=torch.floataa )
        pipe("""This is a test"""	,do_sample=__lowerCamelCase	,top_p=0.5 )






  def        _a							(							self :       Any ):

        '''simple docstring'''




        A_						:  str          =				'''Hello world'''
        A_						:  Optional[int]          =				pipeline("""text-generation"""	,model="""hf-internal-testing/tiny-random-gpt2""" )
        if text_generator.model.framework == "tf":
              A_						:  Tuple          =				logging.get_logger("""transformers.generation.tf_utils""" )
        else:
              A_						:  str          =				logging.get_logger("""transformers.generation.utils""" )
        A_						:  Tuple          =				'''Both `max_new_tokens`'''  # The beggining of the message to be checked in this test

        # Both are set by the user -> log warning
        with CaptureLogger(__lowerCamelCase ) as cl:
              A_						:  Union[str, Any]          =				text_generator(__lowerCamelCase	,max_length=10	,max_new_tokens=1 )
        self.assertIn(__lowerCamelCase	,cl.out )

        # The user only sets one -> no warning
        with CaptureLogger(__lowerCamelCase ) as cl:
              A_						:  Dict          =				text_generator(__lowerCamelCase	,max_new_tokens=1 )
        self.assertNotIn(__lowerCamelCase	,cl.out )

        with CaptureLogger(__lowerCamelCase ) as cl:
              A_						:  Tuple          =				text_generator(__lowerCamelCase	,max_length=10 )
        self.assertNotIn(__lowerCamelCase	,cl.out )




'''simple docstring'''


import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType



class 	__lowerCAmelCase		(							__SCREAMING_SNAKE_CASE    ):


  '''simple docstring'''







  a_  					=							["""image_processor""", """tokenizer"""]
  a_  					=							"""ViltImageProcessor"""
  a_  					=							("""BertTokenizer""", """BertTokenizerFast""")






  def __init__(							self :       List[Any]	,_a :       Optional[Any]=None	,_a :       List[str]=None	,**_a :       Any ):

        '''simple docstring'''




        A_						:  Any          =				None
        if "feature_extractor" in kwargs:
              warnings.warn(
                  """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
                  """ instead."""	,_a	,)
              A_						:  List[str]          =				kwargs.pop("""feature_extractor""" )

        A_						:  List[Any]          =				image_processor if image_processor is not None else feature_extractor
        if image_processor is None:
              raise ValueError("""You need to specify an `image_processor`.""" )
        if tokenizer is None:
              raise ValueError("""You need to specify a `tokenizer`.""" )

        super().__init__(_a	,_a )
        A_						:  Optional[Any]          =				self.image_processor






  def __call__(							self :       Any	,_a :       Tuple	,_a :       Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None	,_a :       bool = True	,_a :       Union[bool, str, PaddingStrategy] = False	,_a :       Union[bool, str, TruncationStrategy] = None	,_a :       Optional[int] = None	,_a :       int = 0	,_a :       Optional[int] = None	,_a :       Optional[bool] = None	,_a :       Optional[bool] = None	,_a :       bool = False	,_a :       bool = False	,_a :       bool = False	,_a :       bool = False	,_a :       bool = True	,_a :       Optional[Union[str, TensorType]] = None	,**_a :       Tuple	,):

        '''simple docstring'''




        A_						:  int          =				self.tokenizer(
            text=_a	,add_special_tokens=_a	,padding=_a	,truncation=_a	,max_length=_a	,stride=_a	,pad_to_multiple_of=_a	,return_token_type_ids=_a	,return_attention_mask=_a	,return_overflowing_tokens=_a	,return_special_tokens_mask=_a	,return_offsets_mapping=_a	,return_length=_a	,verbose=_a	,return_tensors=_a	,**_a	,)
        # add pixel_values + pixel_mask
        A_						:  Optional[int]          =				self.image_processor(_a	,return_tensors=_a )
        encoding.update(_a )

        return encoding






  def        _a							(							self :       List[Any]	,*_a :       Any	,**_a :       Any ):

        '''simple docstring'''




        return self.tokenizer.batch_decode(*_a	,**_a )






  def        _a							(							self :       int	,*_a :       int	,**_a :       Optional[int] ):

        '''simple docstring'''




        return self.tokenizer.decode(*_a	,**_a )






  @property
  def        _a							(							self :       List[Any] ):

        '''simple docstring'''




        A_						:  Optional[int]          =				self.tokenizer.model_input_names
        A_						:  str          =				self.image_processor.model_input_names
        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names ) )






  @property
  def        _a							(							self :       str ):

        '''simple docstring'''




        warnings.warn(
            """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""	,_a	,)
        return self.image_processor_class






  @property
  def        _a							(							self :       int ):

        '''simple docstring'''




        warnings.warn(
            """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""	,_a	,)
        return self.image_processor
