





'''simple docstring'''



import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
	import torch






@is_pipeline_test
class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	a    :				str											=       MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
	a    :				Dict											=       TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							TextaTextGenerationPipeline(model=_lowerCamelCase     ,tokenizer=_lowerCamelCase )
				return generator, ["Something to write", "Something else"]


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							generator('''Something there''' )
				self.assertEqual(_lowerCamelCase     ,[{'''generated_text''': ANY(_lowerCamelCase )}] )
				# These are encoder decoder, they don't just append to incoming string
				self.assertFalse(outputs[0]['''generated_text'''].startswith('''Something there''' ) )

				__lowercase										=							generator(['''This is great !''', '''Something else''']     ,num_return_sequences=2     ,do_sample=_lowerCamelCase )
				self.assertEqual(
				    _lowerCamelCase     ,[
				        [{'''generated_text''': ANY(_lowerCamelCase )}, {'''generated_text''': ANY(_lowerCamelCase )}],
				        [{'''generated_text''': ANY(_lowerCamelCase )}, {'''generated_text''': ANY(_lowerCamelCase )}],
				    ]     ,)

				__lowercase										=							generator(
				    ['''This is great !''', '''Something else''']     ,num_return_sequences=2     ,batch_size=2     ,do_sample=_lowerCamelCase )
				self.assertEqual(
				    _lowerCamelCase     ,[
				        [{'''generated_text''': ANY(_lowerCamelCase )}, {'''generated_text''': ANY(_lowerCamelCase )}],
				        [{'''generated_text''': ANY(_lowerCamelCase )}, {'''generated_text''': ANY(_lowerCamelCase )}],
				    ]     ,)

				with self.assertRaises(_lowerCamelCase ):
							generator(4 )


	@require_torch
	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							pipeline('''text2text-generation'''     ,model='''patrickvonplaten/t5-tiny-random'''     ,framework='''pt''' )
				# do_sample=False necessary for reproducibility
				__lowercase										=							generator('''Something there'''     ,do_sample=_lowerCamelCase )
				self.assertEqual(_lowerCamelCase     ,[{'''generated_text''': ''''''}] )

				__lowercase										=							3
				__lowercase										=							generator(
				    '''Something there'''     ,num_return_sequences=_lowerCamelCase     ,num_beams=_lowerCamelCase     ,)
				__lowercase										=							[
				    {'''generated_text''': '''Beide Beide Beide Beide Beide Beide Beide Beide Beide'''},
				    {'''generated_text''': '''Beide Beide Beide Beide Beide Beide Beide Beide'''},
				    {'''generated_text''': ''''''},
				]
				self.assertEqual(_lowerCamelCase     ,_lowerCamelCase )

				__lowercase										=							generator('''This is a test'''     ,do_sample=_lowerCamelCase     ,num_return_sequences=2     ,return_tensors=_lowerCamelCase )
				self.assertEqual(
				    _lowerCamelCase     ,[
				        {'''generated_token_ids''': ANY(torch.Tensor )},
				        {'''generated_token_ids''': ANY(torch.Tensor )},
				    ]     ,)
				__lowercase										=							generator.model.config.eos_token_id
				__lowercase										=							'''<pad>'''
				__lowercase										=							generator(
				    ['''This is a test''', '''This is a second test''']     ,do_sample=_lowerCamelCase     ,num_return_sequences=2     ,batch_size=2     ,return_tensors=_lowerCamelCase     ,)
				self.assertEqual(
				    _lowerCamelCase     ,[
				        [
				            {'''generated_token_ids''': ANY(torch.Tensor )},
				            {'''generated_token_ids''': ANY(torch.Tensor )},
				        ],
				        [
				            {'''generated_token_ids''': ANY(torch.Tensor )},
				            {'''generated_token_ids''': ANY(torch.Tensor )},
				        ],
				    ]     ,)


	@require_tf
	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				__lowercase										=							pipeline('''text2text-generation'''     ,model='''patrickvonplaten/t5-tiny-random'''     ,framework='''tf''' )
				# do_sample=False necessary for reproducibility
				__lowercase										=							generator('''Something there'''     ,do_sample=_lowerCamelCase )
				self.assertEqual(_lowerCamelCase     ,[{'''generated_text''': ''''''}] )












'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_SCREAMING_SNAKE_CASE					=    logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					=    {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE					=    {
    '''tokenizer_file''': {
        '''EleutherAI/gpt-neox-20b''': '''https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json''',
    },
}

_SCREAMING_SNAKE_CASE					=    {
    '''gpt-neox-20b''': 2_0_4_8,
}






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	a    :				List[Any]											=       VOCAB_FILES_NAMES
	a    :				Optional[Any]											=       PRETRAINED_VOCAB_FILES_MAP
	a    :				Optional[int]											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a    :				List[str]											=       ["input_ids", "attention_mask"]


	def __init__(self     ,_lowerCamelCase=None     ,_lowerCamelCase=None     ,_lowerCamelCase=None     ,_lowerCamelCase="<|endoftext|>"     ,_lowerCamelCase="<|endoftext|>"     ,_lowerCamelCase="<|endoftext|>"     ,_lowerCamelCase=False     ,**_lowerCamelCase     ,)							->							Optional[Any]:







				'''simple docstring'''





				super().__init__(
				    _lowerCamelCase     ,_lowerCamelCase     ,tokenizer_file=_lowerCamelCase     ,unk_token=_lowerCamelCase     ,bos_token=_lowerCamelCase     ,eos_token=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase     ,**_lowerCamelCase     ,)

				__lowercase										=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
				if pre_tok_state.get('''add_prefix_space'''     ,_lowerCamelCase ) != add_prefix_space:
							__lowercase										=							getattr(_lowerCamelCase     ,pre_tok_state.pop('''type''' ) )
							__lowercase										=							add_prefix_space
							__lowercase										=							pre_tok_class(**_lowerCamelCase )

				__lowercase										=							add_prefix_space


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							Tuple[str]:







				'''simple docstring'''





				__lowercase										=							self._tokenizer.model.save(_lowerCamelCase     ,name=_lowerCamelCase )
				return tuple(_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							List[int]:







				'''simple docstring'''





				__lowercase										=							[]
				for is_user, text in conversation.iter_texts():
							input_ids.extend(self.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase ) + [self.eos_token_id] )

				if len(_lowerCamelCase ) > self.model_max_length:
							__lowercase										=							input_ids[-self.model_max_length :]
				return input_ids





