





import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import BertGenerationDecoder, BertGenerationEncoder



class       A__     :



      def __init__(				self ,							A_ ,							A_=13 ,							A_=7 ,							A_=True ,							A_=True ,							A_=99 ,							A_=32 ,							A_=5 ,							A_=4 ,							A_=37 ,							A_="gelu" ,							A_=0.1 ,							A_=0.1 ,							A_=50 ,							A_=0.02 ,							A_=True ,							A_=None ,							):
       '''simple docstring'''




       UpperCamelCase			:      Optional[Any] 			=     parent
       UpperCamelCase			:      List[str] 			=     batch_size
       UpperCamelCase			:      Optional[int] 			=     seq_length
       UpperCamelCase			:      Union[str, Any] 			=     is_training
       UpperCamelCase			:      List[Any] 			=     use_input_mask
       UpperCamelCase			:      str 			=     vocab_size
       UpperCamelCase			:      Union[str, Any] 			=     hidden_size
       UpperCamelCase			:      Union[str, Any] 			=     num_hidden_layers
       UpperCamelCase			:      Union[str, Any] 			=     num_attention_heads
       UpperCamelCase			:      List[str] 			=     intermediate_size
       UpperCamelCase			:      int 			=     hidden_act
       UpperCamelCase			:      Union[str, Any] 			=     hidden_dropout_prob
       UpperCamelCase			:      List[str] 			=     attention_probs_dropout_prob
       UpperCamelCase			:      Tuple 			=     max_position_embeddings
       UpperCamelCase			:      List[Any] 			=     initializer_range
       UpperCamelCase			:      Union[str, Any] 			=     use_labels
       UpperCamelCase			:      str 			=     scope



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size     )

       UpperCamelCase			:      List[str] 			=     None
       if self.use_input_mask:
        UpperCamelCase			:      Dict 			=     random_attention_mask([self.batch_size, self.seq_length]     )

       if self.use_labels:
        UpperCamelCase			:      List[str] 			=     ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size     )

       UpperCamelCase			:      Optional[int] 			=     self.get_config()

       return config, input_ids, input_mask, token_labels



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return BertGenerationConfig(
           vocab_size=self.vocab_size ,							hidden_size=self.hidden_size ,							num_hidden_layers=self.num_hidden_layers ,							num_attention_heads=self.num_attention_heads ,							intermediate_size=self.intermediate_size ,							hidden_act=self.hidden_act ,							hidden_dropout_prob=self.hidden_dropout_prob ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob ,							max_position_embeddings=self.max_position_embeddings ,							is_decoder=A_ ,							initializer_range=self.initializer_range ,							)



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       (
           UpperCamelCase					
       )			:      Dict 			=     self.prepare_config_and_inputs()

       UpperCamelCase			:      Optional[Any] 			=     True
       UpperCamelCase			:      Union[str, Any] 			=     floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
       UpperCamelCase			:      Dict 			=     ids_tensor([self.batch_size, self.seq_length] ,							vocab_size=2     )

       return (
           config,
           input_ids,
           input_mask,
           token_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )



      def  __UpperCamelCase(				self ,							A_ ,							A_ ,							A_ ,							A_ ,							**A_ ,							):
       '''simple docstring'''




       UpperCamelCase			:      Optional[Any] 			=     BertGenerationEncoder(config=A_     )
       model.to(A_     )
       model.eval()
       UpperCamelCase			:      List[str] 			=     model(A_ ,							attention_mask=A_     )
       UpperCamelCase			:      int 			=     model(A_     )
       self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.hidden_size)     )



      def  __UpperCamelCase(				self ,							A_ ,							A_ ,							A_ ,							A_ ,							A_ ,							A_ ,							**A_ ,							):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     True
       UpperCamelCase			:      Tuple 			=     BertGenerationEncoder(config=A_     )
       model.to(A_     )
       model.eval()
       UpperCamelCase			:      Dict 			=     model(
           A_ ,							attention_mask=A_ ,							encoder_hidden_states=A_ ,							encoder_attention_mask=A_ ,							)
       UpperCamelCase			:      Optional[int] 			=     model(
           A_ ,							attention_mask=A_ ,							encoder_hidden_states=A_ ,							)
       self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.hidden_size)     )



      def  __UpperCamelCase(				self ,							A_ ,							A_ ,							A_ ,							A_ ,							A_ ,							A_ ,							**A_ ,							):
       '''simple docstring'''




       UpperCamelCase			:      Union[str, Any] 			=     True
       UpperCamelCase			:      Tuple 			=     True
       UpperCamelCase			:      Optional[int] 			=     BertGenerationDecoder(config=A_     ).to(A_     ).eval()

       # first forward pass
       UpperCamelCase			:      str 			=     model(
           A_ ,							attention_mask=A_ ,							encoder_hidden_states=A_ ,							encoder_attention_mask=A_ ,							use_cache=A_ ,							)
       UpperCamelCase			:      Dict 			=     outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       UpperCamelCase			:      int 			=     ids_tensor((self.batch_size, 3) ,							config.vocab_size     )
       UpperCamelCase			:      Dict 			=     ids_tensor((self.batch_size, 3) ,							vocab_size=2     )

       # append to next input_ids and
       UpperCamelCase			:      Optional[int] 			=     torch.cat([input_ids, next_tokens] ,							dim=-1     )
       UpperCamelCase			:      Dict 			=     torch.cat([input_mask, next_mask] ,							dim=-1     )

       UpperCamelCase			:      List[str] 			=     model(
           A_ ,							attention_mask=A_ ,							encoder_hidden_states=A_ ,							encoder_attention_mask=A_ ,							output_hidden_states=A_ ,							)["hidden_states"][0]
       UpperCamelCase			:      Union[str, Any] 			=     model(
           A_ ,							attention_mask=A_ ,							encoder_hidden_states=A_ ,							encoder_attention_mask=A_ ,							past_key_values=A_ ,							output_hidden_states=A_ ,							)["hidden_states"][0]

       # select random slice
       UpperCamelCase			:      List[Any] 			=     ids_tensor((1,) ,							output_from_past.shape[-1]     ).item()
       UpperCamelCase			:      Dict 			=     output_from_no_past[:, -3:, random_slice_idx].detach()
       UpperCamelCase			:      Optional[Any] 			=     output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(A_ ,							A_ ,							atol=1e-3     )     )



      def  __UpperCamelCase(				self ,							A_ ,							A_ ,							A_ ,							A_ ,							*A_ ,							):
       '''simple docstring'''




       UpperCamelCase			:      Any 			=     BertGenerationDecoder(A_     )
       model.to(A_     )
       model.eval()
       UpperCamelCase			:      Union[str, Any] 			=     model(A_ ,							attention_mask=A_ ,							labels=A_     )
       self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.vocab_size)     )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[Any] 			=     self.prepare_config_and_inputs()
       UpperCamelCase			:      Dict 			=     {"input_ids": input_ids, "attention_mask": input_mask}
       return config, inputs_dict



@require_torch
class       A__     ( __snake_case ,     __snake_case ,     __snake_case ,     unittest.TestCase    ):
      _UpperCAmelCase			:str							=     (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
      _UpperCAmelCase			:Optional[int]							=     (BertGenerationDecoder,) if is_torch_available() else ()
      _UpperCAmelCase			:List[str]							=     (
          {'feature-extraction': BertGenerationEncoder, 'text-generation': BertGenerationDecoder}
          if is_torch_available()
          else {}
      )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     BertGenerationEncoderTester(self     )
       UpperCamelCase			:      Optional[Any] 			=     ConfigTester(self ,							config_class=A_ ,							hidden_size=37     )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       self.config_tester.run_common_tests()



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      int 			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*A_     )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      List[Any] 			=     self.model_tester.prepare_config_and_inputs()
       UpperCamelCase			:      Dict 			=     "bert"
       self.model_tester.create_and_check_model(A_ ,							A_ ,							A_ ,							A_     )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(*A_     )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      str 			=     self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_decoder_model_past_large_inputs(*A_     )



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       (
           UpperCamelCase					
       )			:      Tuple 			=     self.model_tester.prepare_config_and_inputs_for_decoder()

       UpperCamelCase			:      List[str] 			=     None

       self.model_tester.create_and_check_model_as_decoder(
           A_ ,							A_ ,							A_ ,							A_ ,							A_ ,							A_ ,							)



      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_for_causal_lm(*A_     )



      @slow
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      List[Any] 			=     BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"     )
       self.assertIsNotNone(A_     )



@require_torch
class       A__     ( unittest.TestCase    ):



      @slow
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"     )
       UpperCamelCase			:      Tuple 			=     torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 1_0140, 102]]     )
       with torch.no_grad():
        UpperCamelCase			:      Union[str, Any] 			=     model(A_     )[0]
       UpperCamelCase			:      List[str] 			=     torch.Size([1, 8, 1024]     )
       self.assertEqual(output.shape ,							A_     )
       UpperCamelCase			:      int 			=     torch.tensor(
           [[[0.17_75, 0.00_83, -0.03_21], [1.60_02, 0.12_87, 0.39_12], [2.14_73, 0.57_91, 0.60_66]]]     )
       self.assertTrue(torch.allclose(output[:, :3, :3] ,							A_ ,							atol=1e-4     )     )



@require_torch
class       A__     ( unittest.TestCase    ):



      @slow
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      List[str] 			=     BertGenerationDecoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"     )
       UpperCamelCase			:      List[Any] 			=     torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 1_0140, 102]]     )
       with torch.no_grad():
        UpperCamelCase			:      int 			=     model(A_     )[0]
       UpperCamelCase			:      Optional[int] 			=     torch.Size([1, 8, 5_0358]     )
       self.assertEqual(output.shape ,							A_     )
       UpperCamelCase			:      List[Any] 			=     torch.tensor(
           [[[-0.57_88, -2.59_94, -3.70_54], [0.04_38, 4.79_97, 1.87_95], [1.58_62, 6.64_09, 4.46_38]]]     )
       self.assertTrue(torch.allclose(output[:, :3, :3] ,							A_ ,							atol=1e-4     )     )







import argparse
from pathlib import Path

from transformers import AutoConfig, AutoTokenizer, RagConfig, RagSequenceForGeneration, RagTokenForGeneration



def 	A_       (  _lowerCAmelCase							,     _lowerCAmelCase							,     _lowerCAmelCase							,     _lowerCAmelCase							,     _lowerCAmelCase = None							,     _lowerCAmelCase = None							,     _lowerCAmelCase = None							,     )		->				str:
 if config_name_or_path is None:
  UpperCamelCase			:      Dict 			=     "facebook/rag-token-base" if model_type == "rag_token" else "facebook/rag-sequence-base"

 if generator_tokenizer_name_or_path is None:
  UpperCamelCase			:      Tuple 			=     generator_name_or_path

 if question_encoder_tokenizer_name_or_path is None:
  UpperCamelCase			:      Tuple 			=     question_encoder_name_or_path

 UpperCamelCase			:      Any 			=     RagTokenForGeneration if model_type == "rag_token" else RagSequenceForGeneration

 # Save model.
 UpperCamelCase			:      Optional[Any] 			=     RagConfig.from_pretrained(_lowerCAmelCase  )
 UpperCamelCase			:      Union[str, Any] 			=     AutoConfig.from_pretrained(_lowerCAmelCase  )
 UpperCamelCase			:      Tuple 			=     AutoConfig.from_pretrained(_lowerCAmelCase  )

 UpperCamelCase			:      int 			=     gen_config
 UpperCamelCase			:      Dict 			=     question_encoder_config

 UpperCamelCase			:      Tuple 			=     model_class.from_pretrained_question_encoder_generator(
     _lowerCAmelCase							,     _lowerCAmelCase							,     config=_lowerCAmelCase  )
 rag_model.save_pretrained(_lowerCAmelCase  )

 # Sanity check.
 model_class.from_pretrained(_lowerCAmelCase  )

 # Save tokenizers.
 UpperCamelCase			:      Optional[Any] 			=     AutoTokenizer.from_pretrained(_lowerCAmelCase  )
 gen_tokenizer.save_pretrained(dest_dir / "generator_tokenizer/"  )
 UpperCamelCase			:      Union[str, Any] 			=     AutoTokenizer.from_pretrained(_lowerCAmelCase  )
 question_encoder_tokenizer.save_pretrained(dest_dir / "question_encoder_tokenizer/"  )


if __name__ == "__main__":
    __lowerCamelCase :  Tuple										=  argparse.ArgumentParser()
    parser.add_argument(
        """--model_type""",
        choices=["""rag_sequence""", """rag_token"""],
        required=True,
        type=str,
        help="""RAG model type: rag_sequence, rag_token""",
    )
    parser.add_argument("""--dest""", type=str, required=True, help="""Path to the output checkpoint directory.""")
    parser.add_argument("""--generator_name_or_path""", type=str, required=True, help="""Generator model identifier""")
    parser.add_argument(
        """--question_encoder_name_or_path""", type=str, required=True, help="""Question encoder model identifier"""
    )

    parser.add_argument(
        """--generator_tokenizer_name_or_path""",
        type=str,
        help="""Generator tokenizer identifier, if not specified, resolves to ``generator_name_or_path``""",
    )
    parser.add_argument(
        """--question_encoder_tokenizer_name_or_path""",
        type=str,
        help="""Question encoder tokenizer identifier, if not specified, resolves to ``question_encoder_name_or_path``""",
    )
    parser.add_argument(
        """--config_name_or_path""",
        type=str,
        help=(
            """Identifier of the model config to use, if not provided, resolves to a base config for a given"""
            """ ``model_type``"""
        ),
    )

    __lowerCamelCase :  Dict										=  parser.parse_args()

    __lowerCamelCase :  Dict										=  Path(args.dest)
    dest_dir.mkdir(exist_ok=True)

    consolidate(
        args.model_type,
        args.generator_name_or_path,
        args.question_encoder_name_or_path,
        dest_dir,
        args.config_name_or_path,
        args.generator_tokenizer_name_or_path,
        args.question_encoder_tokenizer_name_or_path,
    )
