




import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
  _snake_case            = argparse.ArgumentParser(
      description=(
          "Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"
          " Distillation"
      )
  )
  parser.add_argument("--model_type", default="bert", choices=["bert"])
  parser.add_argument("--model_name", default="bert-base-uncased", type=str)
  parser.add_argument("--dump_checkpoint", default="serialization_dir/tf_bert-base-uncased_0247911.pth", type=str)
  parser.add_argument("--vocab_transform", action="store_true")
  _snake_case            = parser.parse_args()

  if args.model_type == "bert":
    _snake_case            = BertForMaskedLM.from_pretrained(args.model_name)
    _snake_case            = '''bert'''
  else:
    raise ValueError("args.model_type should be \"bert\".")

  _snake_case            = model.state_dict()
  _snake_case            = {}

  for w in ["word_embeddings", "position_embeddings"]:
    _snake_case            = state_dict[f"""{prefix}.embeddings.{w}.weight"""]
  for w in ["weight", "bias"]:
    _snake_case            = state_dict[f"""{prefix}.embeddings.LayerNorm.{w}"""]

  _snake_case            = 0
  for teacher_idx in [0, 2, 4, 7, 9, 11]:
    for w in ["weight", "bias"]:
      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
      ]
      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
      ]
      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
      ]

      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
      ]
      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
      ]

      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
      ]
      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
      ]
      _snake_case            = state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
      ]
    std_idx += 1

  _snake_case            = state_dict['''cls.predictions.decoder.weight''']
  _snake_case            = state_dict['''cls.predictions.bias''']
  if args.vocab_transform:
    for w in ["weight", "bias"]:
      _snake_case            = state_dict[f"""cls.predictions.transform.dense.{w}"""]
      _snake_case            = state_dict[f"""cls.predictions.transform.LayerNorm.{w}"""]

  print(f"""N layers selected for distillation: {std_idx}""")
  print(f"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

  print(f"""Save transferred checkpoint to {args.dump_checkpoint}.""")
  torch.save(compressed_sd, args.dump_checkpoint)










from unittest import TestCase

from datasets import Sequence, Value
from datasets.arrow_dataset import Dataset







class 		lowercase							(     UpperCamelCase__	):

     def       a__			(						self       )		->							Optional[int]:
           return [
               {"col_1": 3, "col_2": "a"},
               {"col_1": 2, "col_2": "b"},
               {"col_1": 1, "col_2": "c"},
               {"col_1": 0, "col_2": "d"},
           ]

     def       a__			(						self       )		->							Optional[int]:
           _A							:  Union[str, Any]    =		{"""col_1""": [3, 2, 1, 0], """col_2""": ["""a""", """b""", """c""", """d"""]}
           return Dataset.from_dict(_a       )

     def       a__			(						self       )		->							Any:
           _A							:  str    =		self._create_example_records()
           _A							:  List[Any]    =		Dataset.from_list(_a       )
           self.assertListEqual(dset.column_names	,  ["""col_1""", """col_2"""]       )
           for i, r in enumerate(_a       ):
                 self.assertDictEqual(_a	,  example_records[i]       )

     def       a__			(						self       )		->							List[str]:
           _A							:  Dict    =		self._create_example_records()
           _A							:  List[str]    =		Dataset.from_list(_a       )
           _A							:  str    =		Dataset.from_dict({k: [r[k] for r in example_records] for k in example_records[0]}       )
           self.assertEqual(dset.info	,  dset_from_dict.info       )

     def       a__			(						self       )		->							str:  # checks what happens with missing columns
           _A							:  List[str]    =		[{"""col_1""": 1}, {"""col_2""": """x"""}]
           _A							:  List[str]    =		Dataset.from_list(_a       )
           self.assertDictEqual(dset[0]	,  {"""col_1""": 1}       )
           self.assertDictEqual(dset[1]	,  {"""col_1""": None}       )  # NB: first record is used for columns

     def       a__			(						self       )		->							Dict:  # checks if the type can be inferred from the second record
           _A							:  List[Any]    =		[{"""col_1""": []}, {"""col_1""": [1, 2]}]
           _A							:  str    =		Dataset.from_list(_a       )
           self.assertEqual(dset.info.features["""col_1"""]	,  Sequence(Value("""int64"""       )       )       )



     def       a__			(						self       )		->							Dict:
           _A							:  List[str]    =		Dataset.from_list([]       )
           self.assertEqual(len(_a       )	,  0       )
           self.assertListEqual(dset.column_names	,  []       )

