






"""simple docstring"""





import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel


class      _lowercase :


      def __init__(    self				:	List[Any]							,   a				:	Dict							,   a				:	str=1_3							,   a				:	str=7							,   a				:	List[Any]=True							,   a				:	Dict=True							,   a				:	Tuple=True							,   a				:	int=True							,   a				:	int=9_9							,   a				:	Optional[int]=3_2							,   a				:	List[str]=5							,   a				:	List[str]=4							,   a				:	Optional[Any]=4							,   a				:	Tuple="gelu"							,   a				:	List[Any]=0.0							,   a				:	str=0.1							,   a				:	List[str]=True							,   a				:	int=5_1_2							,   a				:	Tuple=1_6							,   a				:	Any=2							,   a				:	str=0.0_2							,   a				:	Tuple=3							,   a				:	Any=4							,   a				:	Tuple=None							,   ):





            """simple docstring"""







            __snake_case      :    Dict  			=parent
            __snake_case      :    str  			=batch_size
            __snake_case      :    List[Any]  			=seq_length
            __snake_case      :    List[Any]  			=is_training
            __snake_case      :    Any  			=use_input_mask
            __snake_case      :    List[Any]  			=use_token_type_ids
            __snake_case      :    Optional[Any]  			=use_labels
            __snake_case      :    str  			=vocab_size
            __snake_case      :    List[Any]  			=hidden_size
            __snake_case      :    List[Any]  			=num_hidden_layers
            __snake_case      :    Union[str, Any]  			=num_attention_heads
            __snake_case      :    Union[str, Any]  			=intermediate_multiple_size
            __snake_case      :    Optional[int]  			=hidden_act
            __snake_case      :    str  			=hidden_dropout
            __snake_case      :    str  			=attention_dropout
            __snake_case      :    Dict  			=weight_tying
            __snake_case      :    Optional[int]  			=max_position_embeddings
            __snake_case      :    Any  			=type_vocab_size
            __snake_case      :    Optional[Any]  			=type_sequence_label_size
            __snake_case      :    Dict  			=initializer_range
            __snake_case      :    str  			=num_labels
            __snake_case      :    int  			=num_choices
            __snake_case      :    Optional[Any]  			=scope


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Dict  			=ids_tensor([self.batch_size, self.seq_length]							,   self.vocab_size						)

            __snake_case      :    Tuple  			=None
            if self.use_input_mask:
                  __snake_case      :    Union[str, Any]  			=random_attention_mask([self.batch_size, self.seq_length]						)

            __snake_case      :    str  			=None
            if self.use_labels:
                  __snake_case      :    Dict  			=ids_tensor([self.batch_size, self.seq_length]							,   self.num_labels						)

            __snake_case      :    Any  			=self.get_config()

            return config, input_ids, input_mask, token_labels


      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            return GPTNeoXJapaneseConfig(
                vocab_size=self.vocab_size							,   hidden_size=self.hidden_size							,   num_hidden_layers=self.num_hidden_layers							,   num_attention_heads=self.num_attention_heads							,   intermediate_multiple_size=self.intermediate_multiple_size							,   hidden_act=self.hidden_act							,   hidden_dropout=self.hidden_dropout							,   attention_dropout=self.attention_dropout							,   weight_tying=self.weight_tying							,   max_position_embeddings=self.max_position_embeddings							,   type_vocab_size=self.type_vocab_size							,   is_decoder=a							,   initializer_range=self.initializer_range							,   )


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    Optional[Any]  			=self.prepare_config_and_inputs()

            __snake_case      :    Optional[int]  			=True

            return config, input_ids, input_mask, token_labels


      def _UpperCamelCase						(    self				:	List[Any]							,   a				:	int							,   a				:	Tuple							,   a				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=GPTNeoXJapaneseModel(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Any  			=model(a							,   attention_mask=a						)
            __snake_case      :    int  			=model(a						)
            self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)						)


      def _UpperCamelCase						(    self				:	List[Any]							,   a				:	List[Any]							,   a				:	Optional[Any]							,   a				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    List[str]  			=True
            __snake_case      :    int  			=GPTNeoXJapaneseModel(a						)
            model.to(a						)
            model.eval()
            __snake_case      :    str  			=model(a							,   attention_mask=a						)
            self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)						)


      def _UpperCamelCase						(    self				:	Optional[Any]							,   a				:	List[Any]							,   a				:	int							,   a				:	str							,   a				:	int						):





            """simple docstring"""







            __snake_case      :    Dict  			=GPTNeoXJapaneseForCausalLM(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    List[str]  			=model(a							,   attention_mask=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.vocab_size)						)


      def _UpperCamelCase						(    self				:	Dict							,   a				:	Tuple							,   a				:	List[str]							,   a				:	Tuple						):





            """simple docstring"""







            __snake_case      :    str  			=True
            __snake_case      :    Optional[int]  			=GPTNeoXJapaneseForCausalLM(config=a						)
            model.to(a						)
            model.eval()

            # first forward pass
            __snake_case      :    str  			=model(a							,   attention_mask=a							,   use_cache=a						)
            __snake_case      :    int  			=outputs.past_key_values

            # create hypothetical multiple next token and extent to next_input_ids
            __snake_case      :    List[Any]  			=ids_tensor((self.batch_size, 3)							,   config.vocab_size						)
            __snake_case      :    Optional[Any]  			=ids_tensor((self.batch_size, 3)							,   vocab_size=2						)

            # append to next input_ids and
            __snake_case      :    List[str]  			=torch.cat([input_ids, next_tokens]							,   dim=-1						)
            __snake_case      :    Tuple  			=torch.cat([input_mask, next_mask]							,   dim=-1						)

            __snake_case      :    Optional[Any]  			=model(a							,   attention_mask=a							,   output_hidden_states=a						)
            __snake_case      :    str  			=output_from_no_past['''hidden_states'''][0]
            __snake_case      :    Tuple  			=model(
                a							,   attention_mask=a							,   past_key_values=a							,   output_hidden_states=a							,   )['''hidden_states'''][0]

            # select random slice
            __snake_case      :    Optional[Any]  			=ids_tensor((1,)							,   output_from_past.shape[-1]						).item()
            __snake_case      :    str  			=output_from_no_past[:, -3:, random_slice_idx].detach()
            __snake_case      :    Union[str, Any]  			=output_from_past[:, :, random_slice_idx].detach()

            self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

            # test that outputs are equal for slice
            self.parent.assertTrue(torch.allclose(a							,   a							,   atol=1e-3						)						)







      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            __snake_case      :    Optional[int]  			=self.prepare_config_and_inputs()
            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    List[Any]  			=config_and_inputs
            __snake_case      :    Any  			={'''input_ids''': input_ids, '''attention_mask''': input_mask}
            return config, inputs_dict



@require_torch
class      _lowercase (     lowerCAmelCase		,  lowerCAmelCase		,  unittest.TestCase						):
      _a   :     str								=   (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
      _a   :     Optional[int]								=   (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
      _a   :     Any								=   (
          {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
          if is_torch_available()
          else {}
      )
      _a   :     str								=   False
      _a   :     str								=   False
      _a   :     Tuple								=   False
      _a   :     List[str]								=   False


      def _UpperCamelCase						(    self				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case      :    Any  			=GPTNeoXJapaneseModelTester(self						)
            __snake_case      :    Optional[int]  			=ConfigTester(self							,   config_class=a							,   hidden_size=3_7						)


      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            self.config_tester.run_common_tests()


      def _UpperCamelCase						(    self				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    Optional[Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(a							,   a							,   a						)


      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    str  			=self.model_tester.prepare_config_and_inputs_for_decoder()
            self.model_tester.create_and_check_model_as_decoder(a							,   a							,   a						)


      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    Union[str, Any]  			=self.model_tester.prepare_config_and_inputs_for_decoder()

            __snake_case      :    Any  			=None

            self.model_tester.create_and_check_model_as_decoder(a							,   a							,   a						)


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    str  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_decoder_model_past_large_inputs(a							,   a							,   a						)


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            __snake_case      :    Any  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_causal_lm(*a						)







      @slow
      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            __snake_case      :    Tuple  			='''abeja/gpt-neox-japanese-2.7b'''

            __snake_case      :    str  			=['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

            __snake_case      :    str  			=[
                '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
                '''100年後に必要とされる会社は、「人」が中心の会社です。''',
                '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
                '''国境の長いトンネルを抜けると、そこは雪国だった。''',
                '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
            ]

            __snake_case      :    Tuple  			=GPTNeoXJapaneseTokenizer.from_pretrained(a						)
            __snake_case      :    Dict  			=GPTNeoXJapaneseForCausalLM.from_pretrained(a						)

            __snake_case      :    Tuple  			=[]
            for prompt in prompts:
                  __snake_case      :    Union[str, Any]  			=tokenizer(a							,   return_tensors='''pt'''						).input_ids
                  __snake_case      :    Union[str, Any]  			=model.generate(a							,   max_length=5_0						)
                  __snake_case      :    List[Any]  			=tokenizer.batch_decode(a							,   skip_special_tokens=a						)
                  predicted_outputs += generated_string
            self.assertListEqual(a							,   a						)










"""simple docstring"""





# this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.:
#   python ./utils/get_modified_files.py utils src tests examples
#
# it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered
# since the output of this script is fed into Makefile commands it doesn't print a newline after the results

import re
import subprocess
import sys


UpperCamelCase_	:				int										= subprocess.check_output("""git merge-base main HEAD""".split()).decode("""utf-8""")
UpperCamelCase_	:				Optional[int]										= (
    subprocess.check_output(F'''git diff --diff-filter=d --name-only {fork_point_sha}'''.split()).decode("""utf-8""").split()
)

UpperCamelCase_	:				List[Any]										= """|""".join(sys.argv[1:])
UpperCamelCase_	:				List[str]										= re.compile(rF'''^({joined_dirs}).*?\.py$''')

UpperCamelCase_	:				str										= [x for x in modified_files if regex.match(x)]
print(""" """.join(relevant_modified_files), end="""""")


