





'''simple docstring'''

import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse('1.0.0a'):
   raise Exception('requires fairseq >= 1.0.0a')

logging.set_verbosity_info()
UpperCAmelCase					:		Tuple		=							logging.get_logger(__name__)

UpperCAmelCase					:		Dict		=							'Hello world! cécé herlolip'




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Optional[Any]					     =     FairseqRobertaModel.from_pretrained(lowerCAmelCase_	)
      roberta.eval()  # disable dropout
      _snake_case		:	Optional[int]					     =     roberta.model.encoder.sentence_encoder
      _snake_case		:	Optional[Any]					     =     XLMRobertaConfig(
          vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings			,      hidden_size=roberta.cfg.model.encoder_embed_dim			,      num_hidden_layers=roberta.cfg.model.encoder_layers			,      num_attention_heads=roberta.cfg.model.encoder_attention_heads			,      intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim			,      max_position_embeddings=514			,      type_vocab_size=1			,      layer_norm_eps=1E-5			,      )
      if classification_head:
            _snake_case		:	Dict					     =     roberta.model.classification_heads['''mnli'''].out_proj.weight.shape[0]

      print('''Our RoBERTa config:'''			,      lowerCAmelCase_	)

      _snake_case		:	List[str]					     =     XLMRobertaXLForSequenceClassification(lowerCAmelCase_	) if classification_head else XLMRobertaXLForMaskedLM(lowerCAmelCase_	)
      model.eval()

      # Now let's copy all the weights.
      # Embeddings
      _snake_case		:	int					     =     roberta_sent_encoder.embed_tokens.weight
      _snake_case		:	Union[str, Any]					     =     roberta_sent_encoder.embed_positions.weight
      _snake_case		:	str					     =     torch.zeros_like(
          model.roberta.embeddings.token_type_embeddings.weight	)  # just zero them out b/c RoBERTa doesn't use them.

      _snake_case		:	List[Any]					     =     roberta_sent_encoder.layer_norm.weight
      _snake_case		:	List[Any]					     =     roberta_sent_encoder.layer_norm.bias

      for i in range(config.num_hidden_layers	):
            # Encoder: start of layer
            _snake_case		:	BertLayer					     =     model.roberta.encoder.layer[i]
            _snake_case		:	TransformerSentenceEncoderLayer					     =     roberta_sent_encoder.layers[i]

            _snake_case		:	RobertaAttention					     =     layer.attention
            _snake_case		:	Optional[Any]					     =     roberta_layer.self_attn_layer_norm.weight
            _snake_case		:	List[str]					     =     roberta_layer.self_attn_layer_norm.bias

            # self attention
            _snake_case		:	BertSelfAttention					     =     layer.attention.self
            assert (
                roberta_layer.self_attn.k_proj.weight.data.shape
                == roberta_layer.self_attn.q_proj.weight.data.shape
                == roberta_layer.self_attn.v_proj.weight.data.shape
                == torch.Size((config.hidden_size, config.hidden_size)	)
            )

            _snake_case		:	Optional[Any]					     =     roberta_layer.self_attn.q_proj.weight
            _snake_case		:	int					     =     roberta_layer.self_attn.q_proj.bias
            _snake_case		:	Optional[int]					     =     roberta_layer.self_attn.k_proj.weight
            _snake_case		:	Any					     =     roberta_layer.self_attn.k_proj.bias
            _snake_case		:	List[Any]					     =     roberta_layer.self_attn.v_proj.weight
            _snake_case		:	Any					     =     roberta_layer.self_attn.v_proj.bias

            # self-attention output
            _snake_case		:	BertSelfOutput					     =     layer.attention.output
            assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
            _snake_case		:	Optional[Any]					     =     roberta_layer.self_attn.out_proj.weight
            _snake_case		:	Union[str, Any]					     =     roberta_layer.self_attn.out_proj.bias

            # this one is final layer norm
            _snake_case		:	str					     =     roberta_layer.final_layer_norm.weight
            _snake_case		:	Optional[Any]					     =     roberta_layer.final_layer_norm.bias

            # intermediate
            _snake_case		:	BertIntermediate					     =     layer.intermediate
            assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
            _snake_case		:	str					     =     roberta_layer.fca.weight
            _snake_case		:	List[Any]					     =     roberta_layer.fca.bias

            # output
            _snake_case		:	BertOutput					     =     layer.output
            assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
            _snake_case		:	Optional[int]					     =     roberta_layer.fca.weight
            _snake_case		:	Any					     =     roberta_layer.fca.bias
            # end of layer

      if classification_head:
            _snake_case		:	List[Any]					     =     roberta.model.classification_heads['''mnli'''].dense.weight
            _snake_case		:	Tuple					     =     roberta.model.classification_heads['''mnli'''].dense.bias
            _snake_case		:	int					     =     roberta.model.classification_heads['''mnli'''].out_proj.weight
            _snake_case		:	Any					     =     roberta.model.classification_heads['''mnli'''].out_proj.bias
      else:
            # LM Head
            _snake_case		:	Tuple					     =     roberta.model.encoder.lm_head.dense.weight
            _snake_case		:	Union[str, Any]					     =     roberta.model.encoder.lm_head.dense.bias
            _snake_case		:	int					     =     roberta.model.encoder.lm_head.layer_norm.weight
            _snake_case		:	Dict					     =     roberta.model.encoder.lm_head.layer_norm.bias
            _snake_case		:	int					     =     roberta.model.encoder.lm_head.weight
            _snake_case		:	Union[str, Any]					     =     roberta.model.encoder.lm_head.bias

      # Let's check that we get the same results.
      _snake_case		:	torch.Tensor					     =     roberta.encode(lowerCAmelCase_	).unsqueeze(0	)  # batch of size 1

      _snake_case		:	Any					     =     model(lowerCAmelCase_	)[0]
      if classification_head:
            _snake_case		:	Any					     =     roberta.model.classification_heads['''mnli'''](roberta.extract_features(lowerCAmelCase_	)	)
      else:
            _snake_case		:	Any					     =     roberta.model(lowerCAmelCase_	)[0]
      print(our_output.shape			,      their_output.shape	)
      _snake_case		:	str					     =     torch.max(torch.abs(our_output - their_output	)	).item()
      print(f'''max_absolute_diff = {max_absolute_diff}'''	)  # ~ 1e-7
      _snake_case		:	List[Any]					     =     torch.allclose(lowerCAmelCase_			,      lowerCAmelCase_			,      atol=1E-3	)
      print('''Do both models output the same tensors?'''			,      '''🔥''' if success else '''💩'''	)
      if not success:
            raise Exception('''Something went wRoNg'''	)

      pathlib.Path(lowerCAmelCase_	).mkdir(parents=lowerCAmelCase_			,      exist_ok=lowerCAmelCase_	)
      print(f'''Saving model to {pytorch_dump_folder_path}'''	)
      model.save_pretrained(lowerCAmelCase_	)


if __name__ == "__main__":
   UpperCAmelCase					:		Optional[int]		=							argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--roberta_checkpoint_path', default=None, type=str, required=True, help='Path the official PyTorch dump.'
   )
   parser.add_argument(
       '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
   )
   parser.add_argument(
       '--classification_head', action='store_true', help='Whether to convert a final classification head.'
   )
   UpperCAmelCase					:		Dict		=							parser.parse_args()
   convert_xlm_roberta_xl_checkpoint_to_pytorch(
       args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
   )









'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
   from transformers import (
       AutoConfig,
       BertConfig,
       GPTaConfig,
       TaConfig,
       TFAutoModel,
       TFAutoModelForCausalLM,
       TFAutoModelForMaskedLM,
       TFAutoModelForPreTraining,
       TFAutoModelForQuestionAnswering,
       TFAutoModelForSeqaSeqLM,
       TFAutoModelForSequenceClassification,
       TFAutoModelWithLMHead,
       TFBertForMaskedLM,
       TFBertForPreTraining,
       TFBertForQuestionAnswering,
       TFBertForSequenceClassification,
       TFBertModel,
       TFGPTaLMHeadModel,
       TFRobertaForMaskedLM,
       TFTaForConditionalGeneration,
   )
   from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
   from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
   from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
   from transformers import (
       AutoModel,
       AutoModelForCausalLM,
       AutoModelForMaskedLM,
       AutoModelForPreTraining,
       AutoModelForQuestionAnswering,
       AutoModelForSeqaSeqLM,
       AutoModelForSequenceClassification,
       AutoModelWithLMHead,
       BertForMaskedLM,
       BertForPreTraining,
       BertForQuestionAnswering,
       BertForSequenceClassification,
       BertModel,
       GPTaLMHeadModel,
       RobertaForMaskedLM,
       TaForConditionalGeneration,
   )





@is_pt_tf_cross_test
class lowerCamelCase						(unittest.TestCase					):







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Union[str, Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Any					     =     TFAutoModel.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	str					     =     AutoModel.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Optional[Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Dict					     =     TFAutoModelForPreTraining.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     AutoModelForPreTraining.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	Optional[int]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	List[Any]					     =     TFAutoModelForCausalLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	Tuple					     =     TFAutoModelForCausalLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Optional[int]					     =     AutoModelForCausalLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Optional[Any]					     =     AutoModelForCausalLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	int					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[str]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Union[str, Any]					     =     TFAutoModelForMaskedLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	List[str]					     =     TFAutoModelForMaskedLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	int					     =     AutoModelForMaskedLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Optional[int]					     =     AutoModelForMaskedLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[str]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     TFAutoModelForSeqaSeqLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	List[str]					     =     TFAutoModelForSeqaSeqLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	List[str]					     =     AutoModelForSeqaSeqLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Dict					     =     AutoModelForSeqaSeqLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Any					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Any					     =     TFAutoModelForSequenceClassification.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Dict					     =     AutoModelForSequenceClassification.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Optional[int]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	str					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	str					     =     TFAutoModelForQuestionAnswering.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Union[str, Any]					     =     AutoModelForQuestionAnswering.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	Union[str, Any]					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)

        _snake_case		:	Tuple					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	List[str]					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)

        _snake_case		:	int					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)


