



"""simple docstring"""

import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
			import os

			import jax.numpy as jnp
			from jax import jit

			from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
			from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

			a_			   =					'''0.12'''  # assumed parallelism: 8


if is_torch_available():
			import torch







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  Tuple				,    SCREAMING_SNAKE_CASE__ :  Optional[int]=None					):



					"""simple docstring"""





					if rng is None:
										snake_case_					:		Union[str, Any] 					=					random.Random()

					snake_case_					:		Optional[Any] 					=					1
					for dim in shape:
										total_dims *= dim

					snake_case_					:		List[Any] 					=					[]
					for _ in range(SCREAMING_SNAKE_CASE__					):
										values.append(rng.randint(0				,    vocab_size - 1					)					)

					snake_case_					:		Dict 					=					np.array(SCREAMING_SNAKE_CASE__				,    dtype=jnp.intaa					).reshape(SCREAMING_SNAKE_CASE__					)

					return output







def 							SCREAMING_SNAKE_CASE__      (		SCREAMING_SNAKE_CASE__ :  Union[str, Any]				,    SCREAMING_SNAKE_CASE__ :  List[str]=None					):



					"""simple docstring"""





					snake_case_					:		Tuple 					=					ids_tensor(SCREAMING_SNAKE_CASE__				,    vocab_size=2				,    rng=SCREAMING_SNAKE_CASE__					)
					# make sure that at least one token is attended to for each batch
					snake_case_					:		int 					=					1
					return attn_mask







@require_flax
class    __lowercase     :







						"""simple docstring"""







						_A		:							Optional[int]							       =		None
						_A		:							Tuple							       =		()







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					self.model_tester.prepare_config_and_inputs_for_common()

											# cut to half length & take max batch_size 3
											snake_case_					:		List[Any] 					=					2
											snake_case_					:		Tuple 					=					inputs["""input_ids"""].shape[-1] // 2
											snake_case_					:		Tuple 					=					inputs["""input_ids"""][:max_batch_size, :sequence_length]

											snake_case_					:		int 					=					jnp.ones_like(lowercase__       )
											snake_case_					:		List[str] 					=					attention_mask[:max_batch_size, :sequence_length]

											# generate max 5 tokens
											snake_case_					:		Dict 					=					input_ids.shape[-1] + 5
											if config.eos_token_id is not None and config.pad_token_id is None:
																# hack to allow generate for models such as GPT2 as is done in `generate()`
																snake_case_					:		Union[str, Any] 					=					config.eos_token_id
											return config, input_ids, attention_mask, max_length







						@is_pt_flax_cross_test
						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[Any] 					=					self._get_input_ids_and_config()
											snake_case_					:		Any 					=					False
											snake_case_					:		int 					=					max_length
											snake_case_					:		Dict 					=					0

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Tuple 					=					model_class(lowercase__       )

																snake_case_					:		Any 					=					model_class.__name__[4:]  # Skip the "Flax" at the beginning
																snake_case_					:		Dict 					=					getattr(lowercase__   ,		lowercase__       )
																snake_case_					:		Optional[Any] 					=					pt_model_class(lowercase__       ).eval()
																snake_case_					:		Optional[Any] 					=					load_flax_weights_in_pytorch_model(lowercase__   ,		flax_model.params       )

																snake_case_					:		Tuple 					=					flax_model.generate(lowercase__       ).sequences
																snake_case_					:		Dict 					=					pt_model.generate(torch.tensor(lowercase__   ,		dtype=torch.long       )       )

																if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
																					snake_case_					:		Optional[int] 					=					flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

																self.assertListEqual(pt_generation_outputs.numpy().tolist()   ,		flax_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		str 					=					self._get_input_ids_and_config()
											snake_case_					:		List[str] 					=					False
											snake_case_					:		List[Any] 					=					max_length

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Tuple 					=					model_class(lowercase__       )

																snake_case_					:		Tuple 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		List[str] 					=					jit(model.generate       )
																snake_case_					:		Union[str, Any] 					=					jit_generate(lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		str 					=					self._get_input_ids_and_config()
											snake_case_					:		List[Any] 					=					True
											snake_case_					:		Tuple 					=					max_length

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Optional[Any] 					=					model_class(lowercase__       )

																snake_case_					:		List[Any] 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		List[str] 					=					jit(model.generate       )
																snake_case_					:		Optional[int] 					=					jit_generate(lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[str] 					=					self._get_input_ids_and_config()
											snake_case_					:		Tuple 					=					False
											snake_case_					:		Union[str, Any] 					=					max_length
											snake_case_					:		Optional[Any] 					=					2

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Dict 					=					model_class(lowercase__       )

																snake_case_					:		Optional[int] 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		List[Any] 					=					jit(model.generate       )
																snake_case_					:		Optional[Any] 					=					jit_generate(lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Optional[int] 					=					self._get_input_ids_and_config()
											snake_case_					:		Optional[Any] 					=					False
											snake_case_					:		Tuple 					=					max_length
											snake_case_					:		Union[str, Any] 					=					2
											snake_case_					:		int 					=					2

											for model_class in self.all_generative_model_classes:
																snake_case_					:		int 					=					model_class(lowercase__       )

																snake_case_					:		Optional[int] 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[0]   ,		input_ids.shape[0] * config.num_return_sequences       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					self._get_input_ids_and_config()
											snake_case_					:		List[str] 					=					True
											snake_case_					:		List[Any] 					=					max_length
											snake_case_					:		str 					=					0.8
											snake_case_					:		Optional[Any] 					=					10
											snake_case_					:		str 					=					0.3
											snake_case_					:		str 					=					1
											snake_case_					:		Optional[Any] 					=					8
											snake_case_					:		int 					=					9

											for model_class in self.all_generative_model_classes:
																snake_case_					:		int 					=					model_class(lowercase__       )

																snake_case_					:		Optional[int] 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		Optional[int] 					=					jit(model.generate       )
																snake_case_					:		str 					=					jit_generate(lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Any 					=					self._get_input_ids_and_config()
											snake_case_					:		str 					=					max_length
											snake_case_					:		Optional[Any] 					=					1
											snake_case_					:		Optional[int] 					=					8
											snake_case_					:		Union[str, Any] 					=					9

											for model_class in self.all_generative_model_classes:
																snake_case_					:		str 					=					model_class(lowercase__       )

																snake_case_					:		Any 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		Any 					=					jit(model.generate       )
																snake_case_					:		List[str] 					=					jit_generate(lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self._get_input_ids_and_config()
											snake_case_					:		List[str] 					=					max_length
											snake_case_					:		Optional[int] 					=					2
											snake_case_					:		List[Any] 					=					1
											snake_case_					:		Optional[Any] 					=					8
											snake_case_					:		str 					=					9

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Tuple 					=					model_class(lowercase__       )

																snake_case_					:		Optional[int] 					=					model.generate(lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		List[Any] 					=					jit(model.generate       )
																snake_case_					:		List[str] 					=					jit_generate(lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					self._get_input_ids_and_config()

											# pad attention mask on the left
											snake_case_					:		Tuple 					=					attention_mask.at[(0, 0)].set(0       )

											snake_case_					:		List[str] 					=					False
											snake_case_					:		Tuple 					=					max_length

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Dict 					=					model_class(lowercase__       )

																snake_case_					:		Optional[Any] 					=					model.generate(lowercase__   ,		attention_mask=lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		str 					=					jit(model.generate       )
																snake_case_					:		int 					=					jit_generate(lowercase__   ,		attention_mask=lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Any 					=					self._get_input_ids_and_config()

											# pad attention mask on the left
											snake_case_					:		Tuple 					=					attention_mask.at[(0, 0)].set(0       )

											snake_case_					:		Dict 					=					True
											snake_case_					:		Optional[int] 					=					max_length

											for model_class in self.all_generative_model_classes:
																snake_case_					:		str 					=					model_class(lowercase__       )

																snake_case_					:		Dict 					=					model.generate(lowercase__   ,		attention_mask=lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		Tuple 					=					jit(model.generate       )
																snake_case_					:		List[str] 					=					jit_generate(lowercase__   ,		attention_mask=lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		int 					=					self._get_input_ids_and_config()

											# pad attention mask on the left
											snake_case_					:		Any 					=					attention_mask.at[(0, 0)].set(0       )

											snake_case_					:		Tuple 					=					2
											snake_case_					:		Any 					=					max_length

											for model_class in self.all_generative_model_classes:
																snake_case_					:		Any 					=					model_class(lowercase__       )

																snake_case_					:		Union[str, Any] 					=					model.generate(lowercase__   ,		attention_mask=lowercase__       ).sequences
																self.assertEqual(generation_outputs.shape[-1]   ,		lowercase__       )

																snake_case_					:		List[str] 					=					jit(model.generate       )
																snake_case_					:		Optional[Any] 					=					jit_generate(lowercase__   ,		attention_mask=lowercase__       ).sequences

																self.assertListEqual(generation_outputs.tolist()   ,		jit_generation_outputs.tolist()       )







@require_flax
class    __lowercase     (					unittest.TestCase):







						"""simple docstring"""







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[Any] 					=					AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-bert"""       )
											snake_case_					:		int 					=					FlaxAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""       )

											snake_case_					:		List[Any] 					=					"""Hello world"""
											snake_case_					:		Optional[int] 					=					tokenizer(lowercase__   ,		return_tensors="""np"""       ).input_ids

											# typos are quickly detected (the correct argument is `do_sample`)
											with self.assertRaisesRegex(lowercase__   ,		"""do_samples"""       ):
																model.generate(lowercase__   ,		do_samples=lowercase__       )

											# arbitrary arguments that will not be used anywhere are also not accepted
											with self.assertRaisesRegex(lowercase__   ,		"""foo"""       ):
																snake_case_					:		Optional[int] 					=					{"""foo""": """bar"""}
																model.generate(lowercase__   ,		**lowercase__       )









"""simple docstring"""

from manim import *







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					Rectangle(height=0.5   ,		width=0.5       )
											snake_case_					:		str 					=					Rectangle(height=0.46   ,		width=0.46       ).set_stroke(width=0       )

											snake_case_					:		Optional[Any] 					=					[mem.copy() for i in range(6       )]
											snake_case_					:		str 					=					[mem.copy() for i in range(6       )]
											snake_case_					:		str 					=					VGroup(*lowercase__       ).arrange(lowercase__   ,		buff=0       )
											snake_case_					:		Any 					=					VGroup(*lowercase__       ).arrange(lowercase__   ,		buff=0       )
											snake_case_					:		List[str] 					=					VGroup(lowercase__   ,		lowercase__       ).arrange(lowercase__   ,		buff=0       )
											snake_case_					:		List[Any] 					=					Text("""CPU"""   ,		font_size=24       )
											snake_case_					:		Tuple 					=					Group(lowercase__   ,		lowercase__       ).arrange(lowercase__   ,		buff=0.5   ,		aligned_edge=lowercase__       )
											cpu.move_to([-2.5, -0.5, 0]       )
											self.add(lowercase__       )

											snake_case_					:		List[Any] 					=					[mem.copy() for i in range(4       )]
											snake_case_					:		Tuple 					=					VGroup(*lowercase__       ).arrange(lowercase__   ,		buff=0       )
											snake_case_					:		List[str] 					=					Text("""GPU"""   ,		font_size=24       )
											snake_case_					:		Any 					=					Group(lowercase__   ,		lowercase__       ).arrange(lowercase__   ,		buff=0.5   ,		aligned_edge=lowercase__       )
											gpu.move_to([-1, -1, 0]       )
											self.add(lowercase__       )

											snake_case_					:		Optional[Any] 					=					[mem.copy() for i in range(6       )]
											snake_case_					:		List[Any] 					=					VGroup(*lowercase__       ).arrange(lowercase__   ,		buff=0       )

											snake_case_					:		Dict 					=					Text("""Model"""   ,		font_size=24       )
											snake_case_					:		int 					=					Group(lowercase__   ,		lowercase__       ).arrange(lowercase__   ,		buff=0.5   ,		aligned_edge=lowercase__       )
											model.move_to([3, -1.0, 0]       )
											self.add(lowercase__       )

											snake_case_					:		Dict 					=					[]
											for i, rect in enumerate(lowercase__       ):
																rect.set_stroke(lowercase__       )
																# target = fill.copy().set_fill(YELLOW, opacity=0.7)
																# target.move_to(rect)
																# self.add(target)

																snake_case_					:		List[str] 					=					Rectangle(height=0.46 / 4   ,		width=0.46 / 3       ).set_stroke(width=0.0       ).set_fill(lowercase__   ,		opacity=0.7       )

																if i == 0:
																					cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT       )   ,		buff=0.02   ,		direction=lowercase__       )
																					cpu_target.set_x(cpu_target.get_x() + 0.1       )
																elif i == 3:
																					cpu_target.next_to(cpu_targs[0]   ,		direction=lowercase__   ,		buff=0.0       )
																else:
																					cpu_target.next_to(cpu_targs[i - 1]   ,		direction=lowercase__   ,		buff=0.0       )
																self.add(lowercase__       )
																cpu_targs.append(lowercase__       )

											snake_case_					:		List[str] 					=					[mem.copy() for i in range(6       )]
											snake_case_					:		List[str] 					=					VGroup(*lowercase__       ).arrange(lowercase__   ,		buff=0       )

											snake_case_					:		str 					=					Text("""Loaded Checkpoint"""   ,		font_size=24       )
											snake_case_					:		Any 					=					Group(lowercase__   ,		lowercase__       ).arrange(lowercase__   ,		aligned_edge=lowercase__   ,		buff=0.4       )
											checkpoint.move_to([3, 0.5, 0]       )

											snake_case_					:		Optional[Any] 					=					Square(side_length=2.2       )
											key.move_to([-5, 2, 0]       )

											snake_case_					:		Union[str, Any] 					=					MarkupText(
											    f'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'   ,		font_size=18   ,		)

											key_text.move_to([-5, 2.4, 0]       )

											self.add(lowercase__   ,		lowercase__       )

											snake_case_					:		List[Any] 					=					MarkupText(
											    f'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'   ,		font_size=18   ,		)

											blue_text.next_to(lowercase__   ,		DOWN * 2.4   ,		aligned_edge=key_text.get_left()       )

											snake_case_					:		List[Any] 					=					MarkupText(
											    f'Next, a <i><span fgcolor="{BLUE}">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor="{BLUE}">single shard</span>.'   ,		font_size=24   ,		)
											step_a.move_to([2, 2, 0]       )
											self.play(Write(lowercase__       )   ,		Write(lowercase__       )       )

											self.play(Write(lowercase__   ,		run_time=1       )   ,		Create(lowercase__   ,		run_time=1       )       )

											snake_case_					:		Optional[int] 					=					[]
											snake_case_					:		List[str] 					=					[]
											for i, rect in enumerate(lowercase__       ):
																snake_case_					:		Optional[Any] 					=					fill.copy().set_fill(lowercase__   ,		opacity=0.7       )
																target.move_to(lowercase__       )
																first_animations.append(GrowFromCenter(lowercase__   ,		run_time=1       )       )

																snake_case_					:		List[Any] 					=					target.copy()
																cpu_target.generate_target()
																if i < 5:
																					cpu_target.target.move_to(cpu_left_col_base[i + 1]       )
																else:
																					cpu_target.target.move_to(cpu_right_col_base[i - 5]       )
																second_animations.append(MoveToTarget(lowercase__   ,		run_time=1.5       )       )

											self.play(*lowercase__       )
											self.play(*lowercase__       )
											self.wait()




