




import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
     lowerCAmelCase_					    = argparse.ArgumentParser(
         description=(
             """Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"""
             """ Distillation"""
         )
     )
     parser.add_argument("""--model_type""", default="""bert""", choices=["""bert"""])
     parser.add_argument("""--model_name""", default="""bert-base-uncased""", type=str)
     parser.add_argument("""--dump_checkpoint""", default="""serialization_dir/tf_bert-base-uncased_0247911.pth""", type=str)
     parser.add_argument("""--vocab_transform""", action="""store_true""")
     lowerCAmelCase_					    = parser.parse_args()

     if args.model_type == "bert":
          lowerCAmelCase_					    = BertForMaskedLM.from_pretrained(args.model_name)
          lowerCAmelCase_					    = """bert"""
     else:
          raise ValueError("""args.model_type should be \"bert\".""")

     lowerCAmelCase_					    = model.state_dict()
     lowerCAmelCase_					    = {}

     for w in ["word_embeddings", "position_embeddings"]:
          lowerCAmelCase_					    = state_dict[F"""{prefix}.embeddings.{w}.weight"""]
     for w in ["weight", "bias"]:
          lowerCAmelCase_					    = state_dict[F"""{prefix}.embeddings.LayerNorm.{w}"""]

     lowerCAmelCase_					    = 0
     for teacher_idx in [0, 2, 4, 7, 9, 11]:
          for w in ["weight", "bias"]:
               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
               ]
               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
               ]
               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
               ]

               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
               ]
               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
               ]

               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
               ]
               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
               ]
               lowerCAmelCase_					    = state_dict[
                   F"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
               ]
          std_idx += 1

     lowerCAmelCase_					    = state_dict["""cls.predictions.decoder.weight"""]
     lowerCAmelCase_					    = state_dict["""cls.predictions.bias"""]
     if args.vocab_transform:
          for w in ["weight", "bias"]:
               lowerCAmelCase_					    = state_dict[F"""cls.predictions.transform.dense.{w}"""]
               lowerCAmelCase_					    = state_dict[F"""cls.predictions.transform.LayerNorm.{w}"""]

     print(F"""N layers selected for distillation: {std_idx}""")
     print(F"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

     print(F"""Save transferred checkpoint to {args.dump_checkpoint}.""")
     torch.save(compressed_sd, args.dump_checkpoint)







from abc import ABC, abstractmethod
from argparse import ArgumentParser






class  _lowerCAmelCase     (  _lowercase	):

      @staticmethod
      @abstractmethod
      def 	__magic_name__(   __UpperCAmelCase		):
        raise NotImplementedError()







      @abstractmethod
      def 	__magic_name__(   self		):
        raise NotImplementedError()

