


'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase__    	=     {
    'configuration_albert': ['ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'AlbertConfig', 'AlbertOnnxConfig'],
}

try:
	if not is_sentencepiece_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase__    	=     ['AlbertTokenizer']

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase__    	=     ['AlbertTokenizerFast']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase__    	=     [
	    'ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'AlbertForMaskedLM',
	    'AlbertForMultipleChoice',
	    'AlbertForPreTraining',
	    'AlbertForQuestionAnswering',
	    'AlbertForSequenceClassification',
	    'AlbertForTokenClassification',
	    'AlbertModel',
	    'AlbertPreTrainedModel',
	    'load_tf_weights_in_albert',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase__    	=     [
	    'TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TFAlbertForMaskedLM',
	    'TFAlbertForMultipleChoice',
	    'TFAlbertForPreTraining',
	    'TFAlbertForQuestionAnswering',
	    'TFAlbertForSequenceClassification',
	    'TFAlbertForTokenClassification',
	    'TFAlbertMainLayer',
	    'TFAlbertModel',
	    'TFAlbertPreTrainedModel',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCamelCase__    	=     [
	    'FlaxAlbertForMaskedLM',
	    'FlaxAlbertForMultipleChoice',
	    'FlaxAlbertForPreTraining',
	    'FlaxAlbertForQuestionAnswering',
	    'FlaxAlbertForSequenceClassification',
	    'FlaxAlbertForTokenClassification',
	    'FlaxAlbertModel',
	    'FlaxAlbertPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

	try:
		if not is_sentencepiece_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_albert import AlbertTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_albert_fast import AlbertTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_albert import (
		    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    AlbertForMaskedLM,
		    AlbertForMultipleChoice,
		    AlbertForPreTraining,
		    AlbertForQuestionAnswering,
		    AlbertForSequenceClassification,
		    AlbertForTokenClassification,
		    AlbertModel,
		    AlbertPreTrainedModel,
		    load_tf_weights_in_albert,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_albert import (
		    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFAlbertForMaskedLM,
		    TFAlbertForMultipleChoice,
		    TFAlbertForPreTraining,
		    TFAlbertForQuestionAnswering,
		    TFAlbertForSequenceClassification,
		    TFAlbertForTokenClassification,
		    TFAlbertMainLayer,
		    TFAlbertModel,
		    TFAlbertPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_albert import (
		    FlaxAlbertForMaskedLM,
		    FlaxAlbertForMultipleChoice,
		    FlaxAlbertForPreTraining,
		    FlaxAlbertForQuestionAnswering,
		    FlaxAlbertForSequenceClassification,
		    FlaxAlbertForTokenClassification,
		    FlaxAlbertModel,
		    FlaxAlbertPreTrainedModel,
		)
else:
	import sys

	lowerCamelCase__    	=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)







'''simple docstring'''


import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
						lowerCamelCase__    	=     argparse.ArgumentParser(
						    description=(
						        'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
						        ' Distillation'
						    )
						)
						parser.add_argument('--model_type', default='bert', choices=['bert'])
						parser.add_argument('--model_name', default='bert-base-uncased', type=str)
						parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
						parser.add_argument('--vocab_transform', action='store_true')
						lowerCamelCase__    	=     parser.parse_args()

						if args.model_type == "bert":
												lowerCamelCase__    	=     BertForMaskedLM.from_pretrained(args.model_name)
												lowerCamelCase__    	=     'bert'
						else:
												raise ValueError('args.model_type should be "bert".')

						lowerCamelCase__    	=     model.state_dict()
						lowerCamelCase__    	=     {}

						for w in ["word_embeddings", "position_embeddings"]:
												lowerCamelCase__    	=     state_dict[F'''{prefix}.embeddings.{w}.weight''']
						for w in ["weight", "bias"]:
												lowerCamelCase__    	=     state_dict[F'''{prefix}.embeddings.LayerNorm.{w}''']

						lowerCamelCase__    	=     0
						for teacher_idx in [0, 2, 4, 7, 9, 11]:
												for w in ["weight", "bias"]:
																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
																		]
																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
																		]
																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
																		]

																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
																		]
																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
																		]

																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
																		]
																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
																		]
																		lowerCamelCase__    	=     state_dict[
																		    F'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
																		]
												std_idx += 1

						lowerCamelCase__    	=     state_dict['cls.predictions.decoder.weight']
						lowerCamelCase__    	=     state_dict['cls.predictions.bias']
						if args.vocab_transform:
												for w in ["weight", "bias"]:
																		lowerCamelCase__    	=     state_dict[F'''cls.predictions.transform.dense.{w}''']
																		lowerCamelCase__    	=     state_dict[F'''cls.predictions.transform.LayerNorm.{w}''']

						print(F'''N layers selected for distillation: {std_idx}''')
						print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

						print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
						torch.save(compressed_sd, args.dump_checkpoint)



