



import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("""1.0.0a"""):
      raise Exception("""requires fairseq >= 1.0.0a""")

logging.set_verbosity_info()
UpperCAmelCase_					:   List[str] 							=						logging.get_logger(__name__)

UpperCAmelCase_					:   Union[str, Any] 							=						"""Hello world! cécé herlolip"""






def 							_lowerCAmelCase (  _a   :					str					,   _a   :					str					,   _a   :					bool			)	->			int:
     lowerCAmelCase_					:       Dict    =		FairseqRobertaModel.from_pretrained(_a			)
     roberta.eval()  # disable dropout
     lowerCAmelCase_					:       Optional[Any]    =		roberta.model.encoder.sentence_encoder
     lowerCAmelCase_					:       int    =		XLMRobertaConfig(
         vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings					,   hidden_size=roberta.cfg.model.encoder_embed_dim					,   num_hidden_layers=roberta.cfg.model.encoder_layers					,   num_attention_heads=roberta.cfg.model.encoder_attention_heads					,   intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim					,   max_position_embeddings=5_14					,   type_vocab_size=1					,   layer_norm_eps=1E-5					,   )
     if classification_head:
          lowerCAmelCase_					:       Union[str, Any]    =		roberta.model.classification_heads["""mnli"""].out_proj.weight.shape[0]

     print("""Our RoBERTa config:"""					,   _a			)

     lowerCAmelCase_					:       Tuple    =		XLMRobertaXLForSequenceClassification(_a			) if classification_head else XLMRobertaXLForMaskedLM(_a			)
     model.eval()

     # Now let's copy all the weights.
     # Embeddings
     lowerCAmelCase_					:       int    =		roberta_sent_encoder.embed_tokens.weight
     lowerCAmelCase_					:       Union[str, Any]    =		roberta_sent_encoder.embed_positions.weight
     lowerCAmelCase_					:       List[str]    =		torch.zeros_like(
         model.roberta.embeddings.token_type_embeddings.weight			)  # just zero them out b/c RoBERTa doesn't use them.

     lowerCAmelCase_					:       str    =		roberta_sent_encoder.layer_norm.weight
     lowerCAmelCase_					:       Union[str, Any]    =		roberta_sent_encoder.layer_norm.bias

     for i in range(config.num_hidden_layers			):
          # Encoder: start of layer
          lowerCAmelCase_					:       BertLayer    =		model.roberta.encoder.layer[i]
          lowerCAmelCase_					:       TransformerSentenceEncoderLayer    =		roberta_sent_encoder.layers[i]

          lowerCAmelCase_					:       RobertaAttention    =		layer.attention
          lowerCAmelCase_					:       int    =		roberta_layer.self_attn_layer_norm.weight
          lowerCAmelCase_					:       Dict    =		roberta_layer.self_attn_layer_norm.bias

          # self attention
          lowerCAmelCase_					:       BertSelfAttention    =		layer.attention.self
          assert (
              roberta_layer.self_attn.k_proj.weight.data.shape
              == roberta_layer.self_attn.q_proj.weight.data.shape
              == roberta_layer.self_attn.v_proj.weight.data.shape
              == torch.Size((config.hidden_size, config.hidden_size)			)
          )

          lowerCAmelCase_					:       str    =		roberta_layer.self_attn.q_proj.weight
          lowerCAmelCase_					:       Dict    =		roberta_layer.self_attn.q_proj.bias
          lowerCAmelCase_					:       List[str]    =		roberta_layer.self_attn.k_proj.weight
          lowerCAmelCase_					:       Union[str, Any]    =		roberta_layer.self_attn.k_proj.bias
          lowerCAmelCase_					:       Optional[int]    =		roberta_layer.self_attn.v_proj.weight
          lowerCAmelCase_					:       int    =		roberta_layer.self_attn.v_proj.bias

          # self-attention output
          lowerCAmelCase_					:       BertSelfOutput    =		layer.attention.output
          assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
          lowerCAmelCase_					:       List[Any]    =		roberta_layer.self_attn.out_proj.weight
          lowerCAmelCase_					:       Tuple    =		roberta_layer.self_attn.out_proj.bias

          # this one is final layer norm
          lowerCAmelCase_					:       List[Any]    =		roberta_layer.final_layer_norm.weight
          lowerCAmelCase_					:       List[str]    =		roberta_layer.final_layer_norm.bias

          # intermediate
          lowerCAmelCase_					:       BertIntermediate    =		layer.intermediate
          assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
          lowerCAmelCase_					:       List[Any]    =		roberta_layer.fca.weight
          lowerCAmelCase_					:       Optional[Any]    =		roberta_layer.fca.bias

          # output
          lowerCAmelCase_					:       BertOutput    =		layer.output
          assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
          lowerCAmelCase_					:       Tuple    =		roberta_layer.fca.weight
          lowerCAmelCase_					:       Any    =		roberta_layer.fca.bias
          # end of layer

     if classification_head:
          lowerCAmelCase_					:       str    =		roberta.model.classification_heads["""mnli"""].dense.weight
          lowerCAmelCase_					:       List[Any]    =		roberta.model.classification_heads["""mnli"""].dense.bias
          lowerCAmelCase_					:       Optional[Any]    =		roberta.model.classification_heads["""mnli"""].out_proj.weight
          lowerCAmelCase_					:       Tuple    =		roberta.model.classification_heads["""mnli"""].out_proj.bias
     else:
          # LM Head
          lowerCAmelCase_					:       Union[str, Any]    =		roberta.model.encoder.lm_head.dense.weight
          lowerCAmelCase_					:       Any    =		roberta.model.encoder.lm_head.dense.bias
          lowerCAmelCase_					:       List[Any]    =		roberta.model.encoder.lm_head.layer_norm.weight
          lowerCAmelCase_					:       List[str]    =		roberta.model.encoder.lm_head.layer_norm.bias
          lowerCAmelCase_					:       List[str]    =		roberta.model.encoder.lm_head.weight
          lowerCAmelCase_					:       Dict    =		roberta.model.encoder.lm_head.bias

     # Let's check that we get the same results.
     lowerCAmelCase_					:       torch.Tensor    =		roberta.encode(_a			).unsqueeze(0			)  # batch of size 1

     lowerCAmelCase_					:       Dict    =		model(_a			)[0]
     if classification_head:
          lowerCAmelCase_					:       int    =		roberta.model.classification_heads["""mnli"""](roberta.extract_features(_a			)			)
     else:
          lowerCAmelCase_					:       Dict    =		roberta.model(_a			)[0]
     print(our_output.shape					,   their_output.shape			)
     lowerCAmelCase_					:       Optional[Any]    =		torch.max(torch.abs(our_output - their_output			)			).item()
     print(F'max_absolute_diff = {max_absolute_diff}'			)  # ~ 1e-7
     lowerCAmelCase_					:       Tuple    =		torch.allclose(_a					,   _a					,   atol=1E-3			)
     print("""Do both models output the same tensors?"""					,   """🔥""" if success else """💩"""			)
     if not success:
          raise Exception("""Something went wRoNg"""			)

     pathlib.Path(_a			).mkdir(parents=_a					,   exist_ok=_a			)
     print(F'Saving model to {pytorch_dump_folder_path}'			)
     model.save_pretrained(_a			)


if __name__ == "__main__":
      UpperCAmelCase_					:   Optional[Any] 							=						argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--roberta_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
      )
      parser.add_argument(
          """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
      )
      parser.add_argument(
          """--classification_head""", action="""store_true""", help="""Whether to convert a final classification head."""
      )
      UpperCAmelCase_					:   Optional[Any] 							=						parser.parse_args()
      convert_xlm_roberta_xl_checkpoint_to_pytorch(
          args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
      )







from collections.abc import Generator
from math import sin






def 							_lowerCAmelCase (  _a   :					bytes			)	->			bytes:
     if len(_a			) != 32:
          raise ValueError("""Input must be of length 32"""			)

     lowerCAmelCase_					:       Any    =		B""""""
     for i in [3, 2, 1, 0]:
          little_endian += string_aa[8 * i : 8 * i + 8]
     return little_endian






def 							_lowerCAmelCase (  _a   :					int			)	->			bytes:
     if i < 0:
          raise ValueError("""Input must be non-negative"""			)

     lowerCAmelCase_					:       Tuple    =		format(_a					,   """08x"""			)[-8:]
     lowerCAmelCase_					:       Any    =		B""""""
     for i in [3, 2, 1, 0]:
          little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode("""utf-8"""			)
     return little_endian_hex






def 							_lowerCAmelCase (  _a   :					bytes			)	->			bytes:
     lowerCAmelCase_					:       Tuple    =		B""""""
     for char in message:
          bit_string += format(_a					,   """08b"""			).encode("""utf-8"""			)
     lowerCAmelCase_					:       Dict    =		format(len(_a			)					,   """064b"""			).encode("""utf-8"""			)

     # Pad bit_string to a multiple of 512 chars
     bit_string += b"1"
     while len(_a			) % 5_12 != 4_48:
          bit_string += b"0"
     bit_string += to_little_endian(start_len[32:]			) + to_little_endian(start_len[:32]			)

     return bit_string






def 							_lowerCAmelCase (  _a   :					bytes			)	->			Generator[list[int], None, None]:
     if len(_a			) % 5_12 != 0:
          raise ValueError("""Input must have length that's a multiple of 512"""			)

     for pos in range(0					,   len(_a			)					,   5_12			):
          lowerCAmelCase_					:       int    =		bit_string[pos : pos + 5_12]
          lowerCAmelCase_					:       Any    =		[]
          for i in range(0					,   5_12					,   32			):
               block_words.append(int(to_little_endian(block[i : i + 32]			)					,   2			)			)
          yield block_words






def 							_lowerCAmelCase (  _a   :					int			)	->			int:
     if i < 0:
          raise ValueError("""Input must be non-negative"""			)

     lowerCAmelCase_					:       List[str]    =		format(_a					,   """032b"""			)
     lowerCAmelCase_					:       Optional[int]    =		""""""
     for c in i_str:
          new_str += "1" if c == "0" else "0"
     return int(_a					,   2			)






def 							_lowerCAmelCase (  _a   :					int					,   _a   :					int			)	->			int:
     return (a + b) % 2**32






def 							_lowerCAmelCase (  _a   :					int					,   _a   :					int			)	->			int:
     if i < 0:
          raise ValueError("""Input must be non-negative"""			)
     if shift < 0:
          raise ValueError("""Shift must be non-negative"""			)
     return ((i << shift) ^ (i >> (32 - shift))) % 2**32






def 							_lowerCAmelCase (  _a   :					bytes			)	->			bytes:
     lowerCAmelCase_					:       Union[str, Any]    =		preprocess(_a			)

     lowerCAmelCase_					:       Optional[int]    =		[int(2**32 * abs(sin(i + 1			)			)			) for i in range(64			)]

     # Starting states
     lowerCAmelCase_					:       Tuple    =		0X67452301
     lowerCAmelCase_					:       Optional[int]    =		0XEFCDAB89
     lowerCAmelCase_					:       Tuple    =		0X98BADCFE
     lowerCAmelCase_					:       Tuple    =		0X10325476

     lowerCAmelCase_					:       Dict    =		[
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
     ]

     # Process bit string in chunks, each with 16 32-char words
     for block_words in get_block_words(_a			):
          lowerCAmelCase_					:       int    =		aa
          lowerCAmelCase_					:       Any    =		ba
          lowerCAmelCase_					:       List[str]    =		ca
          lowerCAmelCase_					:       Optional[Any]    =		da

          # Hash current chunk
          for i in range(64			):
               if i <= 15:
                    # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
                    lowerCAmelCase_					:       int    =		d ^ (b & (c ^ d))
                    lowerCAmelCase_					:       List[Any]    =		i
               elif i <= 31:
                    # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
                    lowerCAmelCase_					:       List[Any]    =		c ^ (d & (b ^ c))
                    lowerCAmelCase_					:       Optional[Any]    =		(5 * i + 1) % 16
               elif i <= 47:
                    lowerCAmelCase_					:       Union[str, Any]    =		b ^ c ^ d
                    lowerCAmelCase_					:       Optional[Any]    =		(3 * i + 5) % 16
               else:
                    lowerCAmelCase_					:       Any    =		c ^ (b | not_aa(_a			))
                    lowerCAmelCase_					:       List[str]    =		(7 * i) % 16
               lowerCAmelCase_					:       Tuple    =		(f + a + added_consts[i] + block_words[g]) % 2**32
               lowerCAmelCase_					:       Tuple    =		d
               lowerCAmelCase_					:       Optional[Any]    =		c
               lowerCAmelCase_					:       Dict    =		b
               lowerCAmelCase_					:       List[Any]    =		sum_aa(_a					,   left_rotate_aa(_a					,   shift_amounts[i]			)			)

          # Add hashed chunk to running total
          lowerCAmelCase_					:       Optional[int]    =		sum_aa(_a					,   _a			)
          lowerCAmelCase_					:       Optional[int]    =		sum_aa(_a					,   _a			)
          lowerCAmelCase_					:       Dict    =		sum_aa(_a					,   _a			)
          lowerCAmelCase_					:       Tuple    =		sum_aa(_a					,   _a			)

     lowerCAmelCase_					:       int    =		reformat_hex(_a			) + reformat_hex(_a			) + reformat_hex(_a			) + reformat_hex(_a			)
     return digest


if __name__ == "__main__":
      import doctest

      doctest.testmod()
