
import os
import sys
import unittest


A			      =      os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
A			      =      os.path.join(git_repo_path, 'src', 'transformers')

A			      =      '\n{0} = None\n'

A			      =      '\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n'


A			      =      '\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n'




class     __snake_case		(   unittest.TestCase):
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[Any]									= find_backend('    _import_structure["models.albert"].append("AlbertTokenizerFast")'      )
									self.assertIsNone(A      )

									lowerCamelCase					:			Optional[Any]									= find_backend('    if not is_tokenizers_available():'      )
									self.assertEqual(A,    'tokenizers'      )

									lowerCamelCase					:			Optional[int]									= find_backend('    if not is_tensorflow_text_available():'      )
									self.assertEqual(A,    'tensorflow_text'      )

									lowerCamelCase					:			List[Any]									= find_backend('    if not (is_sentencepiece_available() and is_tokenizers_available()):'      )
									self.assertEqual(A,    'sentencepiece_and_tokenizers'      )

									lowerCamelCase					:			Dict									= find_backend(
									    '    if not (is_sentencepiece_available() and is_tensorflow_text_available()):'      )
									self.assertEqual(A,    'sentencepiece_and_tensorflow_text'      )

									lowerCamelCase					:			Union[str, Any]									= find_backend(
									    '    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):'      )
									self.assertEqual(A,    'sentencepiece_and_tokenizers_and_vision'      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Dict									= read_init()
									# We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
									self.assertIn('torch',    A      )
									self.assertIn('tensorflow_text',    A      )
									self.assertIn('sentencepiece_and_tokenizers',    A      )

									# Likewise, we can't assert on the exact content of a key
									self.assertIn('BertModel',    objects['torch']      )
									self.assertIn('TFBertModel',    objects['tf']      )
									self.assertIn('FlaxBertModel',    objects['flax']      )
									self.assertIn('BertModel',    objects['torch']      )
									self.assertIn('TFBertTokenizer',    objects['tensorflow_text']      )
									self.assertIn('convert_slow_tokenizer',    objects['sentencepiece_and_tokenizers']      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Any									= create_dummy_object('CONSTANT',    '\'torch\''      )
									self.assertEqual(A,    '\nCONSTANT = None\n'      )

									lowerCamelCase					:			Optional[int]									= create_dummy_object('function',    '\'torch\''      )
									self.assertEqual(
									    A,    '\ndef function(*args, **kwargs):\n    requires_backends(function, \'torch\')\n'      )

									lowerCamelCase					:			Optional[int]									= '\nclass FakeClass(metaclass=DummyObject):\n    _backends = \'torch\'\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, \'torch\')\n'
									lowerCamelCase					:			Dict									= create_dummy_object('FakeClass',    '\'torch\''      )
									self.assertEqual(A,    A      )




					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Tuple									= '# This file is autogenerated by the command `make fix-copies`, do not edit.\nfrom ..utils import DummyObject, requires_backends\n\n\nCONSTANT = None\n\n\ndef function(*args, **kwargs):\n    requires_backends(function, ["torch"])\n\n\nclass FakeClass(metaclass=DummyObject):\n    _backends = ["torch"]\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, ["torch"])\n'
									lowerCamelCase					:			Union[str, Any]									= create_dummy_files({'torch': ['CONSTANT', 'function', 'FakeClass']}      )
									self.assertEqual(dummy_files['torch'],    A      )


'''simple docstring'''






import unittest
from queue import Empty
from threading import Thread

from transformers import AutoTokenizer, TextIteratorStreamer, TextStreamer, is_torch_available
from transformers.testing_utils import CaptureStdout, require_torch, torch_device

from ..test_modeling_common import ids_tensor


if is_torch_available():
					import torch

					from transformers import AutoModelForCausalLM




@require_torch
class     __snake_case		(   unittest.TestCase):
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Any									= AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
									lowerCamelCase					:			Optional[int]									= AutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      ).to(A      )
									lowerCamelCase					:			Union[str, Any]									= -1

									lowerCamelCase					:			Optional[Any]									= ids_tensor((1, 5),    vocab_size=model.config.vocab_size      ).to(A      )
									lowerCamelCase					:			List[Any]									= model.generate(A,    max_new_tokens=10,    do_sample=A      )
									lowerCamelCase					:			int									= tokenizer.decode(greedy_ids[0]      )

									with CaptureStdout() as cs:
													lowerCamelCase					:			Optional[int]									= TextStreamer(A      )
													model.generate(A,    max_new_tokens=10,    do_sample=A,    streamer=A      )
									# The greedy text should be printed to stdout, except for the final "\n" in the streamer
									lowerCamelCase					:			int									= cs.out[:-1]

									self.assertEqual(A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			List[str]									= AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
									lowerCamelCase					:			Dict									= AutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      ).to(A      )
									lowerCamelCase					:			List[Any]									= -1

									lowerCamelCase					:			Optional[int]									= ids_tensor((1, 5),    vocab_size=model.config.vocab_size      ).to(A      )
									lowerCamelCase					:			Any									= model.generate(A,    max_new_tokens=10,    do_sample=A      )
									lowerCamelCase					:			Optional[int]									= tokenizer.decode(greedy_ids[0]      )

									lowerCamelCase					:			Union[str, Any]									= TextIteratorStreamer(A      )
									lowerCamelCase					:			Tuple									= {'input_ids': input_ids, 'max_new_tokens': 10, 'do_sample': False, 'streamer': streamer}
									lowerCamelCase					:			List[Any]									= Thread(target=model.generate,    kwargs=A      )
									thread.start()
									lowerCamelCase					:			List[Any]									= ''
									for new_text in streamer:
													streamer_text += new_text

									self.assertEqual(A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[int]									= AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
									lowerCamelCase					:			int									= AutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      ).to(A      )
									lowerCamelCase					:			Optional[Any]									= -1

									lowerCamelCase					:			Union[str, Any]									= ids_tensor((1, 5),    vocab_size=model.config.vocab_size      ).to(A      )
									lowerCamelCase					:			Union[str, Any]									= model.generate(A,    max_new_tokens=10,    do_sample=A      )
									lowerCamelCase					:			int									= greedy_ids[:, input_ids.shape[1] :]
									lowerCamelCase					:			Dict									= tokenizer.decode(new_greedy_ids[0]      )

									with CaptureStdout() as cs:
													lowerCamelCase					:			Tuple									= TextStreamer(A,    skip_prompt=A      )
													model.generate(A,    max_new_tokens=10,    do_sample=A,    streamer=A      )
									# The greedy text should be printed to stdout, except for the final "\n" in the streamer
									lowerCamelCase					:			Optional[int]									= cs.out[:-1]

									self.assertEqual(A,    A      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[Any]									= AutoTokenizer.from_pretrained('distilgpt2'      )
									lowerCamelCase					:			Dict									= AutoModelForCausalLM.from_pretrained('distilgpt2'      ).to(A      )
									lowerCamelCase					:			Dict									= -1

									lowerCamelCase					:			Any									= torch.ones((1, 5),    device=A      ).long() * model.config.bos_token_id
									with CaptureStdout() as cs:
													lowerCamelCase					:			int									= TextStreamer(A,    skip_special_tokens=A      )
													model.generate(A,    max_new_tokens=1,    do_sample=A,    streamer=A      )

									# The prompt contains a special token, so the streamer should not print it. As such, the output text, when
									# re-tokenized, must only contain one token
									lowerCamelCase					:			List[str]									= cs.out[:-1]  # Remove the final "\n"
									lowerCamelCase					:			Optional[Any]									= tokenizer(A,    return_tensors='pt'      )
									self.assertEqual(streamer_text_tokenized.input_ids.shape,    (1, 1)      )




					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
									lowerCamelCase					:			List[str]									= AutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'      ).to(A      )
									lowerCamelCase					:			Any									= -1

									lowerCamelCase					:			int									= ids_tensor((1, 5),    vocab_size=model.config.vocab_size      ).to(A      )
									lowerCamelCase					:			int									= TextIteratorStreamer(A,    timeout=0.001      )
									lowerCamelCase					:			Any									= {'input_ids': input_ids, 'max_new_tokens': 10, 'do_sample': False, 'streamer': streamer}
									lowerCamelCase					:			Optional[Any]									= Thread(target=model.generate,    kwargs=A      )
									thread.start()

									# The streamer will timeout after 0.001 seconds, so an exception will be raised
									with self.assertRaises(A      ):
													lowerCamelCase					:			Dict									= ''
													for new_text in streamer:
																	streamer_text += new_text
