
import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
    lowerCamelCase      =argparse.ArgumentParser(
        description=(
            "Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"
            " Distillation"
        )
    )
    parser.add_argument("--model_type", default="bert", choices=["bert"])
    parser.add_argument("--model_name", default="bert-base-uncased", type=str)
    parser.add_argument("--dump_checkpoint", default="serialization_dir/tf_bert-base-uncased_0247911.pth", type=str)
    parser.add_argument("--vocab_transform", action="store_true")
    lowerCamelCase      =parser.parse_args()

    if args.model_type == "bert":
        lowerCamelCase      =BertForMaskedLM.from_pretrained(args.model_name)
        lowerCamelCase      ="bert"
    else:
        raise ValueError("args.model_type should be \"bert\".")

    lowerCamelCase      =model.state_dict()
    lowerCamelCase      ={}

    for w in ["word_embeddings", "position_embeddings"]:
        lowerCamelCase      =state_dict[F'''{prefix}.embeddings.{w}.weight''']
    for w in ["weight", "bias"]:
        lowerCamelCase      =state_dict[F'''{prefix}.embeddings.LayerNorm.{w}''']

    lowerCamelCase      =0
    for teacher_idx in [0, 2, 4, 7, 9, 1_1]:
        for w in ["weight", "bias"]:
            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
            ]
            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
            ]
            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
            ]

            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
            ]
            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
            ]

            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
            ]
            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
            ]
            lowerCamelCase      =state_dict[
                F'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
            ]
        std_idx += 1

    lowerCamelCase      =state_dict["cls.predictions.decoder.weight"]
    lowerCamelCase      =state_dict["cls.predictions.bias"]
    if args.vocab_transform:
        for w in ["weight", "bias"]:
            lowerCamelCase      =state_dict[F'''cls.predictions.transform.dense.{w}''']
            lowerCamelCase      =state_dict[F'''cls.predictions.transform.LayerNorm.{w}''']

    print(F'''N layers selected for distillation: {std_idx}''')
    print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

    print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
    torch.save(compressed_sd, args.dump_checkpoint)




import doctest
from collections import deque

import numpy as np




class 				_lowerCamelCase :





     """simple docstring"""







     def __init__(							self  )  ->   None:




           """simple docstring"""







           UpperCamelCase__    :   Tuple					=				[2, 1, 2, -1]
           UpperCamelCase__    :   Union[str, Any]					=				[1, 2, 3, 4]






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   list[float]:




           """simple docstring"""







           UpperCamelCase__    :   List[str]					=				len(self.first_signal  )
           UpperCamelCase__    :   Any					=				len(self.second_signal  )

           UpperCamelCase__    :   Dict					=				max(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )

           # create a zero matrix of max_length x max_length
           UpperCamelCase__    :   List[Any]					=				[[0] * max_length for i in range(__SCREAMING_SNAKE_CASE  )]

           # fills the smaller signal with zeros to make both signals of same length
           if length_first_signal < length_second_signal:
                 self.first_signal += [0] * (max_length - length_first_signal)
           elif length_first_signal > length_second_signal:
                 self.second_signal += [0] * (max_length - length_second_signal)
           for i in range(__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   Optional[int]					=				deque(self.second_signal  )
                 rotated_signal.rotate(__SCREAMING_SNAKE_CASE  )
                 for j, item in enumerate(__SCREAMING_SNAKE_CASE  ):
                       matrix[i][j] += item

        # multiply the matrix with the first signal
           UpperCamelCase__    :   Optional[Any]					=				np.matmul(np.transpose(__SCREAMING_SNAKE_CASE  )						,						np.transpose(self.first_signal  )  )

           # rounding-off to two decimal places
           return [round(__SCREAMING_SNAKE_CASE						,						2  ) for i in final_signal]


if __name__ == "__main__":
    doctest.testmod()


