
"""simple docstring"""





import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY



@is_pipeline_test
@require_torch_or_tf
class __lowerCAmelCase				(					unittest.TestCase      ):




			"""simple docstring"""







			snake_case	      =				MODEL_FOR_CAUSAL_LM_MAPPING
			snake_case	      =				TF_MODEL_FOR_CAUSAL_LM_MAPPING




			@require_torch
			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> List[str]:







								"""simple docstring"""
								A_				 =						pipeline(task="text-generation"       ,   model="sshleifer/tiny-ctrl"       ,   framework="pt"	)
								# Using `do_sample=False` to force deterministic output
								A_				 =						text_generator("This is a test"       ,   do_sample=_snake_case	)
								self.assertEqual(
								    _snake_case       ,   [
								        {
								            "generated_text": (
								                "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
								                " oscope. FiliFili@@"
								            )
								        }
								    ]       ,   )

								A_				 =						text_generator(["This is a test", "This is a second test"]	)
								self.assertEqual(
								    _snake_case       ,   [
								        [
								            {
								                "generated_text": (
								                    "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
								                    " oscope. FiliFili@@"
								                )
								            }
								        ],
								        [
								            {
								                "generated_text": (
								                    "This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy"
								                    " oscope. oscope. FiliFili@@"
								                )
								            }
								        ],
								    ]       ,   )

								A_				 =						text_generator("This is a test"       ,   do_sample=_snake_case       ,   num_return_sequences=2       ,   return_tensors=_snake_case	)
								self.assertEqual(
								    _snake_case       ,   [
								        {"generated_token_ids": ANY(_snake_case	)},
								        {"generated_token_ids": ANY(_snake_case	)},
								    ]       ,   )
								A_				 =						text_generator.model.config.eos_token_id
								A_				 =						"<pad>"
								A_				 =						text_generator(
								    ["This is a test", "This is a second test"]       ,   do_sample=_snake_case       ,   num_return_sequences=2       ,   batch_size=2       ,   return_tensors=_snake_case       ,   )
								self.assertEqual(
								    _snake_case       ,   [
								        [
								            {"generated_token_ids": ANY(_snake_case	)},
								            {"generated_token_ids": ANY(_snake_case	)},
								        ],
								        [
								            {"generated_token_ids": ANY(_snake_case	)},
								            {"generated_token_ids": ANY(_snake_case	)},
								        ],
								    ]       ,   )




			@require_tf
			def      lowerCamelCase__     (				self						:					List[str]	)      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						pipeline(task="text-generation"       ,   model="sshleifer/tiny-ctrl"       ,   framework="tf"	)

								# Using `do_sample=False` to force deterministic output
								A_				 =						text_generator("This is a test"       ,   do_sample=_snake_case	)
								self.assertEqual(
								    _snake_case       ,   [
								        {
								            "generated_text": (
								                "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
								                " please,"
								            )
								        }
								    ]       ,   )

								A_				 =						text_generator(["This is a test", "This is a second test"]       ,   do_sample=_snake_case	)
								self.assertEqual(
								    _snake_case       ,   [
								        [
								            {
								                "generated_text": (
								                    "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
								                    " please,"
								                )
								            }
								        ],
								        [
								            {
								                "generated_text": (
								                    "This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes"
								                    " Cannes 閲閲Cannes Cannes Cannes 攵 please,"
								                )
								            }
								        ],
								    ]       ,   )




			def      lowerCamelCase__     (				self						:					str       ,   _snake_case						:					Union[str, Any]       ,   _snake_case						:					Tuple       ,   _snake_case						:					List[Any]	)      -> Tuple:







								"""simple docstring"""
								A_				 =						TextGenerationPipeline(model=_snake_case       ,   tokenizer=_snake_case	)
								return text_generator, ["This is a test", "Another test"]




			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Union[str, Any]:







								"""simple docstring"""
								A_				 =						"Hello I believe in"
								A_				 =						pipeline("text-generation"       ,   model="hf-internal-testing/tiny-random-gpt2"	)
								A_				 =						text_generator(_snake_case	)
								self.assertEqual(
								    _snake_case       ,   [{"generated_text": "Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe"}]       ,   )

								A_				 =						text_generator(_snake_case       ,   stop_sequence=" fe"	)
								self.assertEqual(_snake_case       ,   [{"generated_text": "Hello I believe in fe"}]	)




			def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					int       ,   _snake_case						:					Dict	)      -> Dict:







								"""simple docstring"""
								A_				 =						text_generator.model
								A_				 =						text_generator.tokenizer

								A_				 =						text_generator("This is a test"	)
								self.assertEqual(_snake_case       ,   [{"generated_text": ANY(_snake_case	)}]	)
								self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"	)	)

								A_				 =						text_generator("This is a test"       ,   return_full_text=_snake_case	)
								self.assertEqual(_snake_case       ,   [{"generated_text": ANY(_snake_case	)}]	)
								self.assertNotIn("This is a test"       ,   outputs[0]["generated_text"]	)

								A_				 =						pipeline(task="text-generation"       ,   model=_snake_case       ,   tokenizer=_snake_case       ,   return_full_text=_snake_case	)
								A_				 =						text_generator("This is a test"	)
								self.assertEqual(_snake_case       ,   [{"generated_text": ANY(_snake_case	)}]	)
								self.assertNotIn("This is a test"       ,   outputs[0]["generated_text"]	)

								A_				 =						text_generator("This is a test"       ,   return_full_text=_snake_case	)
								self.assertEqual(_snake_case       ,   [{"generated_text": ANY(_snake_case	)}]	)
								self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"	)	)

								A_				 =						text_generator(["This is great !", "Something else"]       ,   num_return_sequences=2       ,   do_sample=_snake_case	)
								self.assertEqual(
								    _snake_case       ,   [
								        [{"generated_text": ANY(_snake_case	)}, {"generated_text": ANY(_snake_case	)}],
								        [{"generated_text": ANY(_snake_case	)}, {"generated_text": ANY(_snake_case	)}],
								    ]       ,   )

								if text_generator.tokenizer.pad_token is not None:
													A_				 =						text_generator(
													    ["This is great !", "Something else"]       ,   num_return_sequences=2       ,   batch_size=2       ,   do_sample=_snake_case	)
													self.assertEqual(
													    _snake_case       ,   [
													        [{"generated_text": ANY(_snake_case	)}, {"generated_text": ANY(_snake_case	)}],
													        [{"generated_text": ANY(_snake_case	)}, {"generated_text": ANY(_snake_case	)}],
													    ]       ,   )

								with self.assertRaises(_snake_case	):
													A_				 =						text_generator("test"       ,   return_full_text=_snake_case       ,   return_text=_snake_case	)
								with self.assertRaises(_snake_case	):
													A_				 =						text_generator("test"       ,   return_full_text=_snake_case       ,   return_tensors=_snake_case	)
								with self.assertRaises(_snake_case	):
													A_				 =						text_generator("test"       ,   return_text=_snake_case       ,   return_tensors=_snake_case	)

								# Empty prompt is slighly special
								# it requires BOS token to exist.
								# Special case for Pegasus which will always append EOS so will
								# work even without BOS.
								if (
								    text_generator.tokenizer.bos_token_id is not None
								    or "Pegasus" in tokenizer.__class__.__name__
								    or "Git" in model.__class__.__name__
								):
													A_				 =						text_generator(""	)
													self.assertEqual(_snake_case       ,   [{"generated_text": ANY(_snake_case	)}]	)
								else:
													with self.assertRaises((ValueError, AssertionError)	):
																		A_				 =						text_generator(""	)

								if text_generator.framework == "tf":
													# TF generation does not support max_new_tokens, and it's impossible
													# to control long generation with only max_length without
													# fancy calculation, dismissing tests for now.
													return
								# We don't care about infinite range models.
								# They already work.
								# Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
								A_				 =						["RwkvForCausalLM", "XGLMForCausalLM", "GPTNeoXForCausalLM"]
								if (
								    tokenizer.model_max_length < 10_000
								    and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
								):
													# Handling of large generations
													with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)	):
																		text_generator("This is a test" * 500       ,   max_new_tokens=20	)

													A_				 =						text_generator("This is a test" * 500       ,   handle_long_generation="hole"       ,   max_new_tokens=20	)
													# Hole strategy cannot work
													with self.assertRaises(_snake_case	):
																		text_generator(
																		    "This is a test" * 500       ,   handle_long_generation="hole"       ,   max_new_tokens=tokenizer.model_max_length + 10       ,   )




			@require_torch
			@require_accelerate
			@require_torch_gpu
			def      lowerCamelCase__     (				self						:					Optional[int]	)      -> Any:







								"""simple docstring"""
								import torch

								# Classic `model_kwargs`
								A_				 =						pipeline(
								    model="hf-internal-testing/tiny-random-bloom"       ,   model_kwargs={"device_map": "auto", "torch_dtype": torch.bfloataa}       ,   )
								self.assertEqual(pipe.model.device       ,   torch.device(0	)	)
								self.assertEqual(pipe.model.lm_head.weight.dtype       ,   torch.bfloataa	)
								A_				 =						pipe("This is a test"	)
								self.assertEqual(
								    _snake_case       ,   [
								        {
								            "generated_text": (
								                "This is a test test test test test test test test test test test test test test test test"
								                " test"
								            )
								        }
								    ]       ,   )

								# Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
								A_				 =						pipeline(model="hf-internal-testing/tiny-random-bloom"       ,   device_map="auto"       ,   torch_dtype=torch.bfloataa	)
								self.assertEqual(pipe.model.device       ,   torch.device(0	)	)
								self.assertEqual(pipe.model.lm_head.weight.dtype       ,   torch.bfloataa	)
								A_				 =						pipe("This is a test"	)
								self.assertEqual(
								    _snake_case       ,   [
								        {
								            "generated_text": (
								                "This is a test test test test test test test test test test test test test test test test"
								                " test"
								            )
								        }
								    ]       ,   )

								# torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
								A_				 =						pipeline(model="hf-internal-testing/tiny-random-bloom"       ,   device_map="auto"	)
								self.assertEqual(pipe.model.device       ,   torch.device(0	)	)
								self.assertEqual(pipe.model.lm_head.weight.dtype       ,   torch.floataa	)
								A_				 =						pipe("This is a test"	)
								self.assertEqual(
								    _snake_case       ,   [
								        {
								            "generated_text": (
								                "This is a test test test test test test test test test test test test test test test test"
								                " test"
								            )
								        }
								    ]       ,   )




			@require_torch
			@require_torch_gpu
			def      lowerCamelCase__     (				self						:					Dict	)      -> str:







								"""simple docstring"""
								import torch

								A_				 =						pipeline(model="hf-internal-testing/tiny-random-bloom"       ,   device=0       ,   torch_dtype=torch.floataa	)
								pipe("This is a test"	)




			@require_torch
			@require_accelerate
			@require_torch_gpu
			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> List[str]:







								"""simple docstring"""
								import torch

								A_				 =						pipeline(model="hf-internal-testing/tiny-random-bloom"       ,   device_map="auto"       ,   torch_dtype=torch.floataa	)
								pipe("This is a test"       ,   do_sample=_snake_case       ,   top_p=0.5	)






			def      lowerCamelCase__     (				self						:					str	)      -> List[str]:







								"""simple docstring"""
								A_				 =						"Hello world"
								A_				 =						pipeline("text-generation"       ,   model="hf-internal-testing/tiny-random-gpt2"	)
								if text_generator.model.framework == "tf":
													A_				 =						logging.get_logger("transformers.generation.tf_utils"	)
								else:
													A_				 =						logging.get_logger("transformers.generation.utils"	)
								A_				 =						"Both `max_new_tokens`"  # The beggining of the message to be checked in this test

								# Both are set by the user -> log warning
								with CaptureLogger(_snake_case	) as cl:
													A_				 =						text_generator(_snake_case       ,   max_length=10       ,   max_new_tokens=1	)
								self.assertIn(_snake_case       ,   cl.out	)

								# The user only sets one -> no warning
								with CaptureLogger(_snake_case	) as cl:
													A_				 =						text_generator(_snake_case       ,   max_new_tokens=1	)
								self.assertNotIn(_snake_case       ,   cl.out	)

								with CaptureLogger(_snake_case	) as cl:
													A_				 =						text_generator(_snake_case       ,   max_length=10	)
								self.assertNotIn(_snake_case       ,   cl.out	)






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCamelCase_					:						Any				= logging.get_logger(__name__)

UpperCamelCase_					:						Any				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						List[str]				= {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_					:						Union[str, Any]				= {
    '''yjernite/retribert-base-uncased''': 512,
}


UpperCamelCase_					:						str				= {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				RetriBertTokenizer
			snake_case	      =				["input_ids", "attention_mask"]




			def __init__(				self						:					Tuple       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=None       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]="[UNK]"       ,   _snake_case						:					Optional[Any]="[SEP]"       ,   _snake_case						:					str="[PAD]"       ,   _snake_case						:					List[str]="[CLS]"       ,   _snake_case						:					List[str]="[MASK]"       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					Tuple       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					Union[str, Any]=None	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)




