



'''simple docstring'''



import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel
class 	lowerCamelCase_							:







						def __init__(  self   :      List[Any]				,    lowerCAmelCase__   :      List[str]				,    lowerCAmelCase__   :      Union[str, Any]=13				,    lowerCAmelCase__   :      Dict=7				,    lowerCAmelCase__   :      Optional[Any]=True				,    lowerCAmelCase__   :      Optional[int]=True				,    lowerCAmelCase__   :      Optional[int]=True				,    lowerCAmelCase__   :      List[str]=True				,    lowerCAmelCase__   :      Any=99				,    lowerCAmelCase__   :      Any=32				,    lowerCAmelCase__   :      List[str]=5				,    lowerCAmelCase__   :      Optional[int]=4				,    lowerCAmelCase__   :      str=4				,    lowerCAmelCase__   :      Optional[int]="gelu"				,    lowerCAmelCase__   :      Dict=0.0				,    lowerCAmelCase__   :      Tuple=0.1				,    lowerCAmelCase__   :      Optional[int]=True				,    lowerCAmelCase__   :      int=5_12				,    lowerCAmelCase__   :      Optional[int]=16				,    lowerCAmelCase__   :      Any=2				,    lowerCAmelCase__   :      Union[str, Any]=0.02				,    lowerCAmelCase__   :      Dict=3				,    lowerCAmelCase__   :      Optional[Any]=4				,    lowerCAmelCase__   :      List[str]=None				,    ):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			parent
													SCREAMING_SNAKE_CASE					:      List[str]     					=			batch_size
													SCREAMING_SNAKE_CASE					:      str     					=			seq_length
													SCREAMING_SNAKE_CASE					:      Any     					=			is_training
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			use_input_mask
													SCREAMING_SNAKE_CASE					:      List[str]     					=			use_token_type_ids
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			use_labels
													SCREAMING_SNAKE_CASE					:      str     					=			vocab_size
													SCREAMING_SNAKE_CASE					:      Dict     					=			hidden_size
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			num_hidden_layers
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			num_attention_heads
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			intermediate_multiple_size
													SCREAMING_SNAKE_CASE					:      Dict     					=			hidden_act
													SCREAMING_SNAKE_CASE					:      str     					=			hidden_dropout
													SCREAMING_SNAKE_CASE					:      Dict     					=			attention_dropout
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			weight_tying
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			max_position_embeddings
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			type_vocab_size
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			type_sequence_label_size
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			initializer_range
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			num_labels
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			num_choices
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			scope







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			ids_tensor([self.batch_size, self.seq_length]				,    self.vocab_size				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			None
													if self.use_input_mask:
																				SCREAMING_SNAKE_CASE					:      Dict     					=			random_attention_mask([self.batch_size, self.seq_length]				)

													SCREAMING_SNAKE_CASE					:      int     					=			None
													if self.use_labels:
																				SCREAMING_SNAKE_CASE					:      List[str]     					=			ids_tensor([self.batch_size, self.seq_length]				,    self.num_labels				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_config()

													return config, input_ids, input_mask, token_labels







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													return GPTNeoXJapaneseConfig(
													    vocab_size=self.vocab_size				,    hidden_size=self.hidden_size				,    num_hidden_layers=self.num_hidden_layers				,    num_attention_heads=self.num_attention_heads				,    intermediate_multiple_size=self.intermediate_multiple_size				,    hidden_act=self.hidden_act				,    hidden_dropout=self.hidden_dropout				,    attention_dropout=self.attention_dropout				,    weight_tying=self.weight_tying				,    max_position_embeddings=self.max_position_embeddings				,    type_vocab_size=self.type_vocab_size				,    is_decoder=lowerCAmelCase__				,    initializer_range=self.initializer_range				,    )







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Tuple     					=			self.prepare_config_and_inputs()

													SCREAMING_SNAKE_CASE					:      List[str]     					=			True

													return config, input_ids, input_mask, token_labels







						def 					__lowercase    (  self   :      Optional[Any]				,    lowerCAmelCase__   :      List[str]				,    lowerCAmelCase__   :      List[str]				,    lowerCAmelCase__   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			GPTNeoXJapaneseModel(config=lowerCAmelCase__				)
													model.to(lowerCAmelCase__				)
													model.eval()
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			model(lowerCAmelCase__				,    attention_mask=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			model(lowerCAmelCase__				)
													self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size)				)







						def 					__lowercase    (  self   :      List[Any]				,    lowerCAmelCase__   :      Optional[Any]				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			True
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			GPTNeoXJapaneseModel(lowerCAmelCase__				)
													model.to(lowerCAmelCase__				)
													model.eval()
													SCREAMING_SNAKE_CASE					:      Dict     					=			model(lowerCAmelCase__				,    attention_mask=lowerCAmelCase__				)
													self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size)				)







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      List[str]				,    lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      Tuple				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			GPTNeoXJapaneseForCausalLM(config=lowerCAmelCase__				)
													model.to(lowerCAmelCase__				)
													model.eval()
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			model(lowerCAmelCase__				,    attention_mask=lowerCAmelCase__				,    labels=lowerCAmelCase__				)
													self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.vocab_size)				)







						def 					__lowercase    (  self   :      Optional[Any]				,    lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      Optional[int]				,    lowerCAmelCase__   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			True
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			GPTNeoXJapaneseForCausalLM(config=lowerCAmelCase__				)
													model.to(lowerCAmelCase__				)
													model.eval()

													# first forward pass
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			model(lowerCAmelCase__				,    attention_mask=lowerCAmelCase__				,    use_cache=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			outputs.past_key_values

													# create hypothetical multiple next token and extent to next_input_ids
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			ids_tensor((self.batch_size, 3)				,    config.vocab_size				)
													SCREAMING_SNAKE_CASE					:      str     					=			ids_tensor((self.batch_size, 3)				,    vocab_size=2				)

													# append to next input_ids and
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			torch.cat([input_ids, next_tokens]				,    dim=-1				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			torch.cat([input_mask, next_mask]				,    dim=-1				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			model(lowerCAmelCase__				,    attention_mask=lowerCAmelCase__				,    output_hidden_states=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			output_from_no_past['''hidden_states'''][0]
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			model(
													    lowerCAmelCase__				,    attention_mask=lowerCAmelCase__				,    past_key_values=lowerCAmelCase__				,    output_hidden_states=lowerCAmelCase__				,    )['''hidden_states'''][0]

													# select random slice
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			ids_tensor((1,)				,    output_from_past.shape[-1]				).item()
													SCREAMING_SNAKE_CASE					:      Tuple     					=			output_from_no_past[:, -3:, random_slice_idx].detach()
													SCREAMING_SNAKE_CASE					:      Tuple     					=			output_from_past[:, :, random_slice_idx].detach()

													self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

													# test that outputs are equal for slice
													self.parent.assertTrue(torch.allclose(lowerCAmelCase__				,    lowerCAmelCase__				,    atol=1e-3				)				)







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			self.prepare_config_and_inputs()
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      int     					=			config_and_inputs
													SCREAMING_SNAKE_CASE					:      int     					=			{'''input_ids''': input_ids, '''attention_mask''': input_mask}
													return config, inputs_dict


@require_torch
class 	lowerCamelCase_							(					snake_case_    ,				snake_case_    ,				unittest.TestCase			):
						_lowerCAmelCase     :   str	     =				(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
						_lowerCAmelCase     :   str	     =				(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
						_lowerCAmelCase     :   Dict	     =				(
						    {'feature-extraction': GPTNeoXJapaneseModel, 'text-generation': GPTNeoXJapaneseForCausalLM}
						    if is_torch_available()
						    else {}
						)
						_lowerCAmelCase     :   Any	     =				False
						_lowerCAmelCase     :   str	     =				False
						_lowerCAmelCase     :   Any	     =				False
						_lowerCAmelCase     :   List[str]	     =				False







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			GPTNeoXJapaneseModelTester(self				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			ConfigTester(self				,    config_class=lowerCAmelCase__				,    hidden_size=37				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													self.config_tester.run_common_tests()







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      str     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      int     					=			self.model_tester.prepare_config_and_inputs_for_decoder()
													self.model_tester.create_and_check_model_as_decoder(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													# This regression test was failing with PyTorch < 1.3
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      int     					=			self.model_tester.prepare_config_and_inputs_for_decoder()

													SCREAMING_SNAKE_CASE					:      Dict     					=			None

													self.model_tester.create_and_check_model_as_decoder(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Dict     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_decoder_model_past_large_inputs(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_causal_lm(*lowerCAmelCase__				)







						@slow
						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''abeja/gpt-neox-japanese-2.7b'''

													SCREAMING_SNAKE_CASE					:      Any     					=			['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

													SCREAMING_SNAKE_CASE					:      Tuple     					=			[
													    '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
													    '''100年後に必要とされる会社は、「人」が中心の会社です。''',
													    '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
													    '''国境の長いトンネルを抜けると、そこは雪国だった。''',
													    '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
													]

													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			GPTNeoXJapaneseTokenizer.from_pretrained(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			GPTNeoXJapaneseForCausalLM.from_pretrained(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			[]
													for prompt in prompts:
																				SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer(lowerCAmelCase__				,    return_tensors='''pt'''				).input_ids
																				SCREAMING_SNAKE_CASE					:      List[str]     					=			model.generate(lowerCAmelCase__				,    max_length=50				)
																				SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			tokenizer.batch_decode(lowerCAmelCase__				,    skip_special_tokens=lowerCAmelCase__				)
																				predicted_outputs += generated_string
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)









'''simple docstring'''



import os


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_doctest_list.py
lowerCAmelCase_     :       Optional[Any]								=						'.'


if __name__ == "__main__":
							lowerCAmelCase_     :       Optional[Any]								=						os.path.join(REPO_PATH, 'utils/documentation_tests.txt')
							lowerCAmelCase_     :       Dict								=						[]
							lowerCAmelCase_     :       Any								=						[]
							with open(doctest_file_path) as fp:
														for line in fp:
																					lowerCAmelCase_     :       List[str]								=						line.strip()
																					lowerCAmelCase_     :       Optional[Any]								=						os.path.join(REPO_PATH, line)
																					if not (os.path.isfile(path) or os.path.isdir(path)):
																												non_existent_paths.append(line)
																					all_paths.append(path)
							if len(non_existent_paths) > 0:
														lowerCAmelCase_     :       Dict								=						'\n'.join(non_existent_paths)
														raise ValueError(f'`utils/documentation_tests.txt` contains non-existent paths:\n{non_existent_paths}')
							if all_paths != sorted(all_paths):
														raise ValueError('Files in `utils/documentation_tests.txt` are not in alphabetical order.')




