



"""simple docstring"""





import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


lowercase  							=					5_00_00
lowercase  							=					50_00

lowercase							,       lowercase  							=					os.path.split(__file__)
lowercase  							=					os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))




@get_duration
def 	UpperCAmelCase     (		A	:						datasets.Dataset     ,				A	:						Optional[Any]       ):

		'''simple docstring'''


		for i in range(A       ):
				_UpperCAmelCase							 =		dataset[i]




@get_duration
def 	UpperCAmelCase     (		A	:						datasets.Dataset     ,				A	:						Optional[Any]     ,				A	:						List[Any]       ):

		'''simple docstring'''


		for i in range(0     ,				len(A       )     ,				A       ):
				_UpperCAmelCase							 =		dataset[i : i + batch_size]




@get_duration
def 	UpperCAmelCase     (		A	:						datasets.Dataset     ,				A	:						List[Any]     ,				A	:						Dict       ):

		'''simple docstring'''


		with dataset.formatted_as(type=A       ):
				for i in range(A       ):
						_UpperCAmelCase							 =		dataset[i]




@get_duration
def 	UpperCAmelCase     (		A	:						datasets.Dataset     ,				A	:						Union[str, Any]     ,				A	:						int     ,				A	:						Optional[Any]       ):

		'''simple docstring'''


		with dataset.formatted_as(type=A       ):
				for i in range(0     ,				A     ,				A       ):
						_UpperCAmelCase							 =		dataset[i : i + batch_size]




def 	UpperCAmelCase     (		):

		'''simple docstring'''


		_UpperCAmelCase							 =		{'num examples': SPEED_TEST_N_EXAMPLES}
		_UpperCAmelCase							 =		[
		    (read, {'length': SMALL_TEST}),
		    (read, {'length': SPEED_TEST_N_EXAMPLES}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1000}),
		    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
		    (read_formatted, {'type': 'pandas', 'length': SMALL_TEST}),
		    (read_formatted, {'type': 'torch', 'length': SMALL_TEST}),
		    (read_formatted, {'type': 'tensorflow', 'length': SMALL_TEST}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1000}),
		]

		_UpperCAmelCase							 =		[
		    (read, {'length': SMALL_TEST}),
		    (read, {'length': SPEED_TEST_N_EXAMPLES}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1000}),
		    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1000}),
		]
		with tempfile.TemporaryDirectory() as tmp_dir:
				print('generating dataset'       )
				_UpperCAmelCase							 =		datasets.Features(
				    {'list': datasets.Sequence(datasets.Value('float32'       )       ), 'numbers': datasets.Value('float32'       )}       )
				_UpperCAmelCase							 =		generate_example_dataset(
				    os.path.join(A     ,				'dataset.arrow'       )     ,				A     ,				num_examples=A     ,				seq_shapes={'list': (100,)}     ,				)
				print('first set of iterations'       )
				for func, kwargs in functions:
						print(func.__name__     ,				str(A       )       )
						_UpperCAmelCase							 =		func(A     ,				**A       )

				print('shuffling dataset'       )
				_UpperCAmelCase							 =		dataset.shuffle()
				print('Second set of iterations (after shuffling'       )
				for func, kwargs in functions_shuffled:
						print('shuffled '     ,				func.__name__     ,				str(A       )       )
						_UpperCAmelCase							 =		func(
						    A     ,				**A       )

		with open(A     ,				'wb'       ) as f:
				f.write(json.dumps(A       ).encode('utf-8'       )       )


if __name__ == "__main__":  # useful to run the profiler
					benchmark_iterating()







"""simple docstring"""





import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
					import torch




@is_pipeline_test
class 			lowercase__			(      unittest.TestCase    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
							_UpperCAmelCase      				=					TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING






							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case	,       snake_case						)			->							Dict:
									_UpperCAmelCase							 =		TextaTextGenerationPipeline(model=snake_case	,       tokenizer=snake_case						)
									return generator, ["Something to write", "Something else"]






							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case						)			->							Dict:
									_UpperCAmelCase							 =		generator('Something there'						)
									self.assertEqual(snake_case	,       [{'generated_text': ANY(snake_case						)}]						)
									# These are encoder decoder, they don't just append to incoming string
									self.assertFalse(outputs[0]['generated_text'].startswith('Something there'						)						)

									_UpperCAmelCase							 =		generator(['This is great !', 'Something else']	,       num_return_sequences=2	,       do_sample=snake_case						)
									self.assertEqual(
									    snake_case	,       [
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									    ]	,       )

									_UpperCAmelCase							 =		generator(
									    ['This is great !', 'Something else']	,       num_return_sequences=2	,       batch_size=2	,       do_sample=snake_case						)
									self.assertEqual(
									    snake_case	,       [
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									        [{'generated_text': ANY(snake_case						)}, {'generated_text': ANY(snake_case						)}],
									    ]	,       )

									with self.assertRaises(snake_case						):
											generator(4						)






							@require_torch
							def        lowerCamelCase_   (  self						)			->							Dict:
									_UpperCAmelCase							 =		pipeline('text2text-generation'	,       model='patrickvonplaten/t5-tiny-random'	,       framework='pt'						)
									# do_sample=False necessary for reproducibility
									_UpperCAmelCase							 =		generator('Something there'	,       do_sample=snake_case						)
									self.assertEqual(snake_case	,       [{'generated_text': ''}]						)

									_UpperCAmelCase							 =		3
									_UpperCAmelCase							 =		generator(
									    'Something there'	,       num_return_sequences=snake_case	,       num_beams=snake_case	,       )
									_UpperCAmelCase							 =		[
									    {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide Beide'},
									    {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide'},
									    {'generated_text': ''},
									]
									self.assertEqual(snake_case	,       snake_case						)

									_UpperCAmelCase							 =		generator('This is a test'	,       do_sample=snake_case	,       num_return_sequences=2	,       return_tensors=snake_case						)
									self.assertEqual(
									    snake_case	,       [
									        {'generated_token_ids': ANY(torch.Tensor						)},
									        {'generated_token_ids': ANY(torch.Tensor						)},
									    ]	,       )
									_UpperCAmelCase							 =		generator.model.config.eos_token_id
									_UpperCAmelCase							 =		'<pad>'
									_UpperCAmelCase							 =		generator(
									    ['This is a test', 'This is a second test']	,       do_sample=snake_case	,       num_return_sequences=2	,       batch_size=2	,       return_tensors=snake_case	,       )
									self.assertEqual(
									    snake_case	,       [
									        [
									            {'generated_token_ids': ANY(torch.Tensor						)},
									            {'generated_token_ids': ANY(torch.Tensor						)},
									        ],
									        [
									            {'generated_token_ids': ANY(torch.Tensor						)},
									            {'generated_token_ids': ANY(torch.Tensor						)},
									        ],
									    ]	,       )






							@require_tf
							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		pipeline('text2text-generation'	,       model='patrickvonplaten/t5-tiny-random'	,       framework='tf'						)
									# do_sample=False necessary for reproducibility
									_UpperCAmelCase							 =		generator('Something there'	,       do_sample=snake_case						)
									self.assertEqual(snake_case	,       [{'generated_text': ''}]						)


