'''simple docstring'''





import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY





@is_pipeline_test
@require_torch_or_tf
class       lowercase__		(     unittest.TestCase):
						UpperCamelCase_  =	MODEL_FOR_CAUSAL_LM_MAPPING
						UpperCamelCase_  =	TF_MODEL_FOR_CAUSAL_LM_MAPPING


						@require_torch
						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							pipeline(task='''text-generation'''					,	model='''sshleifer/tiny-ctrl'''					,	framework='''pt'''		)
												# Using `do_sample=False` to force deterministic output
												SCREAMING_SNAKE_CASE							:					Any           =							text_generator('''This is a test'''					,	do_sample=lowerCamelCase__		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        {
												            '''generated_text''': (
												                '''This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'''
												                ''' oscope. FiliFili@@'''
												            )
												        }
												    ]					,	)

												SCREAMING_SNAKE_CASE							:					Dict           =							text_generator(['''This is a test''', '''This is a second test''']		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        [
												            {
												                '''generated_text''': (
												                    '''This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'''
												                    ''' oscope. FiliFili@@'''
												                )
												            }
												        ],
												        [
												            {
												                '''generated_text''': (
												                    '''This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy'''
												                    ''' oscope. oscope. FiliFili@@'''
												                )
												            }
												        ],
												    ]					,	)

												SCREAMING_SNAKE_CASE							:					int           =							text_generator('''This is a test'''					,	do_sample=lowerCamelCase__					,	num_return_sequences=2					,	return_tensors=lowerCamelCase__		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        {'''generated_token_ids''': ANY(lowerCamelCase__		)},
												        {'''generated_token_ids''': ANY(lowerCamelCase__		)},
												    ]					,	)
												SCREAMING_SNAKE_CASE							:					Any           =							text_generator.model.config.eos_token_id
												SCREAMING_SNAKE_CASE							:					List[Any]           =							"<pad>"
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							text_generator(
												    ['''This is a test''', '''This is a second test''']					,	do_sample=lowerCamelCase__					,	num_return_sequences=2					,	batch_size=2					,	return_tensors=lowerCamelCase__					,	)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        [
												            {'''generated_token_ids''': ANY(lowerCamelCase__		)},
												            {'''generated_token_ids''': ANY(lowerCamelCase__		)},
												        ],
												        [
												            {'''generated_token_ids''': ANY(lowerCamelCase__		)},
												            {'''generated_token_ids''': ANY(lowerCamelCase__		)},
												        ],
												    ]					,	)


						@require_tf
						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[int]           =							pipeline(task='''text-generation'''					,	model='''sshleifer/tiny-ctrl'''					,	framework='''tf'''		)

												# Using `do_sample=False` to force deterministic output
												SCREAMING_SNAKE_CASE							:					Dict           =							text_generator('''This is a test'''					,	do_sample=lowerCamelCase__		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        {
												            '''generated_text''': (
												                '''This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'''
												                ''' please,'''
												            )
												        }
												    ]					,	)

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							text_generator(['''This is a test''', '''This is a second test''']					,	do_sample=lowerCamelCase__		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        [
												            {
												                '''generated_text''': (
												                    '''This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'''
												                    ''' please,'''
												                )
												            }
												        ],
												        [
												            {
												                '''generated_text''': (
												                    '''This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes'''
												                    ''' Cannes 閲閲Cannes Cannes Cannes 攵 please,'''
												                )
												            }
												        ],
												    ]					,	)


						def  __A ( self			:       Union[str, Any]					,	UpperCamelCase__			:       List[Any]					,	UpperCamelCase__			:       Dict					,	UpperCamelCase__			:       List[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							TextGenerationPipeline(model=lowerCamelCase__					,	tokenizer=lowerCamelCase__		)
												return text_generator, ["This is a test", "Another test"]


						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							"Hello I believe in"
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							pipeline('''text-generation'''					,	model='''hf-internal-testing/tiny-random-gpt2'''		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							text_generator(lowerCamelCase__		)
												self.assertEqual(
												    lowerCamelCase__					,	[{'''generated_text''': '''Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe'''}]					,	)

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							text_generator(lowerCamelCase__					,	stop_sequence=''' fe'''		)
												self.assertEqual(lowerCamelCase__					,	[{'''generated_text''': '''Hello I believe in fe'''}]		)


						def  __A ( self			:       Optional[Any]					,	UpperCamelCase__			:       Any					,	UpperCamelCase__			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					List[str]           =							text_generator.model
												SCREAMING_SNAKE_CASE							:					List[str]           =							text_generator.tokenizer

												SCREAMING_SNAKE_CASE							:					List[Any]           =							text_generator('''This is a test'''		)
												self.assertEqual(lowerCamelCase__					,	[{'''generated_text''': ANY(lowerCamelCase__		)}]		)
												self.assertTrue(outputs[0]['''generated_text'''].startswith('''This is a test'''		)		)

												SCREAMING_SNAKE_CASE							:					Dict           =							text_generator('''This is a test'''					,	return_full_text=lowerCamelCase__		)
												self.assertEqual(lowerCamelCase__					,	[{'''generated_text''': ANY(lowerCamelCase__		)}]		)
												self.assertNotIn('''This is a test'''					,	outputs[0]['''generated_text''']		)

												SCREAMING_SNAKE_CASE							:					Optional[int]           =							pipeline(task='''text-generation'''					,	model=lowerCamelCase__					,	tokenizer=lowerCamelCase__					,	return_full_text=lowerCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Any           =							text_generator('''This is a test'''		)
												self.assertEqual(lowerCamelCase__					,	[{'''generated_text''': ANY(lowerCamelCase__		)}]		)
												self.assertNotIn('''This is a test'''					,	outputs[0]['''generated_text''']		)

												SCREAMING_SNAKE_CASE							:					Dict           =							text_generator('''This is a test'''					,	return_full_text=lowerCamelCase__		)
												self.assertEqual(lowerCamelCase__					,	[{'''generated_text''': ANY(lowerCamelCase__		)}]		)
												self.assertTrue(outputs[0]['''generated_text'''].startswith('''This is a test'''		)		)

												SCREAMING_SNAKE_CASE							:					Any           =							text_generator(['''This is great !''', '''Something else''']					,	num_return_sequences=2					,	do_sample=lowerCamelCase__		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        [{'''generated_text''': ANY(lowerCamelCase__		)}, {'''generated_text''': ANY(lowerCamelCase__		)}],
												        [{'''generated_text''': ANY(lowerCamelCase__		)}, {'''generated_text''': ANY(lowerCamelCase__		)}],
												    ]					,	)

												if text_generator.tokenizer.pad_token is not None:
																		SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							text_generator(
																		    ['''This is great !''', '''Something else''']					,	num_return_sequences=2					,	batch_size=2					,	do_sample=lowerCamelCase__		)
																		self.assertEqual(
																		    lowerCamelCase__					,	[
																		        [{'''generated_text''': ANY(lowerCamelCase__		)}, {'''generated_text''': ANY(lowerCamelCase__		)}],
																		        [{'''generated_text''': ANY(lowerCamelCase__		)}, {'''generated_text''': ANY(lowerCamelCase__		)}],
																		    ]					,	)

												with self.assertRaises(lowerCamelCase__		):
																		SCREAMING_SNAKE_CASE							:					Tuple           =							text_generator('''test'''					,	return_full_text=lowerCamelCase__					,	return_text=lowerCamelCase__		)
												with self.assertRaises(lowerCamelCase__		):
																		SCREAMING_SNAKE_CASE							:					Tuple           =							text_generator('''test'''					,	return_full_text=lowerCamelCase__					,	return_tensors=lowerCamelCase__		)
												with self.assertRaises(lowerCamelCase__		):
																		SCREAMING_SNAKE_CASE							:					Dict           =							text_generator('''test'''					,	return_text=lowerCamelCase__					,	return_tensors=lowerCamelCase__		)

												# Empty prompt is slighly special
												# it requires BOS token to exist.
												# Special case for Pegasus which will always append EOS so will
												# work even without BOS.
												if (
												    text_generator.tokenizer.bos_token_id is not None
												    or "Pegasus" in tokenizer.__class__.__name__
												    or "Git" in model.__class__.__name__
												):
																		SCREAMING_SNAKE_CASE							:					List[Any]           =							text_generator(''''''		)
																		self.assertEqual(lowerCamelCase__					,	[{'''generated_text''': ANY(lowerCamelCase__		)}]		)
												else:
																		with self.assertRaises((ValueError, AssertionError)		):
																								SCREAMING_SNAKE_CASE							:					int           =							text_generator(''''''		)

												if text_generator.framework == "tf":
																		# TF generation does not support max_new_tokens, and it's impossible
																		# to control long generation with only max_length without
																		# fancy calculation, dismissing tests for now.
																		return
												# We don't care about infinite range models.
												# They already work.
												# Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
												SCREAMING_SNAKE_CASE							:					Dict           =							["RwkvForCausalLM", "XGLMForCausalLM", "GPTNeoXForCausalLM"]
												if (
												    tokenizer.model_max_length < 1_0000
												    and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
												):
																		# Handling of large generations
																		with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)		):
																								text_generator('''This is a test''' * 500					,	max_new_tokens=20		)

																		SCREAMING_SNAKE_CASE							:					Any           =							text_generator('''This is a test''' * 500					,	handle_long_generation='''hole'''					,	max_new_tokens=20		)
																		# Hole strategy cannot work
																		with self.assertRaises(lowerCamelCase__		):
																								text_generator(
																								    '''This is a test''' * 500					,	handle_long_generation='''hole'''					,	max_new_tokens=tokenizer.model_max_length + 10					,	)


						@require_torch
						@require_accelerate
						@require_torch_gpu
						def  __A ( self			:       str		):





												'''simple docstring'''





												import torch

												# Classic `model_kwargs`
												SCREAMING_SNAKE_CASE							:					Any           =							pipeline(
												    model='''hf-internal-testing/tiny-random-bloom'''					,	model_kwargs={'''device_map''': '''auto''', '''torch_dtype''': torch.bfloataa}					,	)
												self.assertEqual(pipe.model.device					,	torch.device(0		)		)
												self.assertEqual(pipe.model.lm_head.weight.dtype					,	torch.bfloataa		)
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							pipe('''This is a test'''		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        {
												            '''generated_text''': (
												                '''This is a test test test test test test test test test test test test test test test test'''
												                ''' test'''
												            )
												        }
												    ]					,	)

												# Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
												SCREAMING_SNAKE_CASE							:					List[str]           =							pipeline(model='''hf-internal-testing/tiny-random-bloom'''					,	device_map='''auto'''					,	torch_dtype=torch.bfloataa		)
												self.assertEqual(pipe.model.device					,	torch.device(0		)		)
												self.assertEqual(pipe.model.lm_head.weight.dtype					,	torch.bfloataa		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							pipe('''This is a test'''		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        {
												            '''generated_text''': (
												                '''This is a test test test test test test test test test test test test test test test test'''
												                ''' test'''
												            )
												        }
												    ]					,	)

												# torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
												SCREAMING_SNAKE_CASE							:					Dict           =							pipeline(model='''hf-internal-testing/tiny-random-bloom'''					,	device_map='''auto'''		)
												self.assertEqual(pipe.model.device					,	torch.device(0		)		)
												self.assertEqual(pipe.model.lm_head.weight.dtype					,	torch.floataa		)
												SCREAMING_SNAKE_CASE							:					int           =							pipe('''This is a test'''		)
												self.assertEqual(
												    lowerCamelCase__					,	[
												        {
												            '''generated_text''': (
												                '''This is a test test test test test test test test test test test test test test test test'''
												                ''' test'''
												            )
												        }
												    ]					,	)


						@require_torch
						@require_torch_gpu
						def  __A ( self			:       Optional[Any]		):





												'''simple docstring'''





												import torch

												SCREAMING_SNAKE_CASE							:					str           =							pipeline(model='''hf-internal-testing/tiny-random-bloom'''					,	device=0					,	torch_dtype=torch.floataa		)
												pipe('''This is a test'''		)


						@require_torch
						@require_accelerate
						@require_torch_gpu
						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												import torch

												SCREAMING_SNAKE_CASE							:					Dict           =							pipeline(model='''hf-internal-testing/tiny-random-bloom'''					,	device_map='''auto'''					,	torch_dtype=torch.floataa		)
												pipe('''This is a test'''					,	do_sample=lowerCamelCase__					,	top_p=0.5		)



						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Dict           =							"Hello world"
												SCREAMING_SNAKE_CASE							:					int           =							pipeline('''text-generation'''					,	model='''hf-internal-testing/tiny-random-gpt2'''		)
												if text_generator.model.framework == "tf":
																		SCREAMING_SNAKE_CASE							:					Optional[int]           =							logging.get_logger('''transformers.generation.tf_utils'''		)
												else:
																		SCREAMING_SNAKE_CASE							:					Optional[Any]           =							logging.get_logger('''transformers.generation.utils'''		)
												SCREAMING_SNAKE_CASE							:					str           =							"Both `max_new_tokens`"  # The beggining of the message to be checked in this test

												# Both are set by the user -> log warning
												with CaptureLogger(lowerCamelCase__		) as cl:
																		SCREAMING_SNAKE_CASE							:					Tuple           =							text_generator(lowerCamelCase__					,	max_length=10					,	max_new_tokens=1		)
												self.assertIn(lowerCamelCase__					,	cl.out		)

												# The user only sets one -> no warning
												with CaptureLogger(lowerCamelCase__		) as cl:
																		SCREAMING_SNAKE_CASE							:					Optional[Any]           =							text_generator(lowerCamelCase__					,	max_new_tokens=1		)
												self.assertNotIn(lowerCamelCase__					,	cl.out		)

												with CaptureLogger(lowerCamelCase__		) as cl:
																		SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							text_generator(lowerCamelCase__					,	max_length=10		)
												self.assertNotIn(lowerCamelCase__					,	cl.out		)






from __future__ import annotations

import math
from collections import Counter
from string import ascii_lowercase
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE       ,						SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							analyze_text(_lowercase		)
						SCREAMING_SNAKE_CASE							:					Any           =							list(''' ''' + ascii_lowercase		)
						# what is our total sum of probabilities.
						SCREAMING_SNAKE_CASE							:					Tuple           =							sum(single_char_strings.values()		)

						# one length string
						SCREAMING_SNAKE_CASE							:					Tuple           =							0
						# for each alpha we go in our dict and if it is in it we calculate entropy
						for ch in my_alphas:
												if ch in single_char_strings:
																		SCREAMING_SNAKE_CASE							:					Tuple           =							single_char_strings[ch]
																		SCREAMING_SNAKE_CASE							:					List[str]           =							my_str / all_sum
																		my_fir_sum += prob * math.loga(_lowercase		)  # entropy formula.

    # print entropy
						print(f"""{round(-1 * my_fir_sum		):.1f}"""		)

						# two len string
						SCREAMING_SNAKE_CASE							:					Optional[Any]           =							sum(two_char_strings.values()		)
						SCREAMING_SNAKE_CASE							:					List[str]           =							0
						# for each alpha (two in size) calculate entropy.
						for cha in my_alphas:
												for cha in my_alphas:
																		SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							cha + cha
																		if sequence in two_char_strings:
																								SCREAMING_SNAKE_CASE							:					Any           =							two_char_strings[sequence]
																								SCREAMING_SNAKE_CASE							:					Dict           =							int(_lowercase		) / all_sum
																								my_sec_sum += prob * math.loga(_lowercase		)

    # print second entropy
						print(f"""{round(-1 * my_sec_sum		):.1f}"""		)

						# print the difference between them
						print(f"""{round((-1 * my_sec_sum) - (-1 * my_fir_sum)		):.1f}"""		)
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE							:					Tuple           =							Counter()  # type: ignore
						SCREAMING_SNAKE_CASE							:					Any           =							Counter()  # type: ignore
						single_char_strings[text[-1]] += 1

						# first case when we have space at start.
						two_char_strings[" " + text[0]] += 1
						for i in range(0	,						len(_lowercase		) - 1		):
												single_char_strings[text[i]] += 1
												two_char_strings[text[i : i + 2]] += 1
						return single_char_strings, two_char_strings
def    A			(					):
						import doctest

						doctest.testmod()
						# text = (
						#     "Had repulsive dashwoods suspicion sincerity but advantage now him. Remark "
						#     "easily garret nor nay. Civil those mrs enjoy shy fat merry. You greatest "
						#     "jointure saw horrible. He private he on be imagine suppose. Fertile "
						#     "beloved evident through no service elderly is. Blind there if every no so "
						#     "at. Own neglected you preferred way sincerity delivered his attempted. To "
						#     "of message cottage windows do besides against uncivil.  Delightful "
						#     "unreserved impossible few estimating men favourable see entreaties. She "
						#     "propriety immediate was improving. He or entrance humoured likewise "
						#     "moderate. Much nor game son say feel. Fat make met can must form into "
						#     "gate. Me we offending prevailed discovery. "
						# )

						# calculate_prob(text)


if __name__ == "__main__":
					main()





