





import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
	_lowerCamelCase   				=			argparse.ArgumentParser(
	    description=(
	        'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
	        ' Distillation'
	    )
	)
	parser.add_argument('--model_type', default='bert', choices=['bert'])
	parser.add_argument('--model_name', default='bert-base-uncased', type=str)
	parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
	parser.add_argument('--vocab_transform', action='store_true')
	_lowerCamelCase   				=			parser.parse_args()

	if args.model_type == "bert":
		_lowerCamelCase   				=			BertForMaskedLM.from_pretrained(args.model_name)
		_lowerCamelCase   				=			'bert'
	else:
		raise ValueError('args.model_type should be "bert".')

	_lowerCamelCase   				=			model.state_dict()
	_lowerCamelCase   				=			{}

	for w in ["word_embeddings", "position_embeddings"]:
		_lowerCamelCase   				=			state_dict[F'''{prefix}.embeddings.{w}.weight''']
	for w in ["weight", "bias"]:
		_lowerCamelCase   				=			state_dict[F'''{prefix}.embeddings.LayerNorm.{w}''']

	_lowerCamelCase   				=			0
	for teacher_idx in [0, 2, 4, 7, 9, 11]:
		for w in ["weight", "bias"]:
			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
			]
			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
			]
			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
			]

			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
			]
			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
			]

			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
			]
			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
			]
			_lowerCamelCase   				=			state_dict[
			    F'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
			]
		std_idx += 1

	_lowerCamelCase   				=			state_dict['cls.predictions.decoder.weight']
	_lowerCamelCase   				=			state_dict['cls.predictions.bias']
	if args.vocab_transform:
		for w in ["weight", "bias"]:
			_lowerCamelCase   				=			state_dict[F'''cls.predictions.transform.dense.{w}''']
			_lowerCamelCase   				=			state_dict[F'''cls.predictions.transform.LayerNorm.{w}''']

	print(F'''N layers selected for distillation: {std_idx}''')
	print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

	print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
	torch.save(compressed_sd, args.dump_checkpoint)






import unittest

from datasets import load_dataset

from transformers.pipelines import pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_torch, slow




@is_pipeline_test
@require_torch
class   UpperCamelCase_       (     unittest.TestCase    ):


							@require_torch
							def      _snake_case  (							self    :Dict		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      pipeline(
									    task="""zero-shot-audio-classification"""							,				model="""hf-internal-testing/tiny-clap-htsat-unfused"""		)
									SCREAMING_SNAKE_CASE__			=      load_dataset("""ashraq/esc50"""		)
									SCREAMING_SNAKE_CASE__			=      dataset["""train"""]["""audio"""][-1]["""array"""]
									SCREAMING_SNAKE_CASE__			=      audio_classifier(__A							,				candidate_labels=["""Sound of a dog""", """Sound of vaccum cleaner"""]		)
									self.assertEqual(
									    nested_simplify(__A		)							,				[{"""score""": 0.5_0_1, """label""": """Sound of a dog"""}, {"""score""": 0.4_9_9, """label""": """Sound of vaccum cleaner"""}]							,				)


							@unittest.skip("""No models are available in TF"""		)
							def      _snake_case  (							self    :Dict		)   ->   List[str]:



									"""simple docstring"""



									pass


							@slow
							@require_torch
							def      _snake_case  (							self    :Any		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      pipeline(
									    task="""zero-shot-audio-classification"""							,				model="""laion/clap-htsat-unfused"""							,				)
									# This is an audio of a dog
									SCREAMING_SNAKE_CASE__			=      load_dataset("""ashraq/esc50"""		)
									SCREAMING_SNAKE_CASE__			=      dataset["""train"""]["""audio"""][-1]["""array"""]
									SCREAMING_SNAKE_CASE__			=      audio_classifier(__A							,				candidate_labels=["""Sound of a dog""", """Sound of vaccum cleaner"""]		)

									self.assertEqual(
									    nested_simplify(__A		)							,				[
									        {"""score""": 0.9_9_9, """label""": """Sound of a dog"""},
									        {"""score""": 0.0_0_1, """label""": """Sound of vaccum cleaner"""},
									    ]							,				)

									SCREAMING_SNAKE_CASE__			=      audio_classifier([audio] * 5							,				candidate_labels=["""Sound of a dog""", """Sound of vaccum cleaner"""]		)
									self.assertEqual(
									    nested_simplify(__A		)							,				[
									        [
									            {"""score""": 0.9_9_9, """label""": """Sound of a dog"""},
									            {"""score""": 0.0_0_1, """label""": """Sound of vaccum cleaner"""},
									        ],
									    ]
									    * 5							,				)
									SCREAMING_SNAKE_CASE__			=      audio_classifier(
									    [audio] * 5							,				candidate_labels=["""Sound of a dog""", """Sound of vaccum cleaner"""]							,				batch_size=5		)
									self.assertEqual(
									    nested_simplify(__A		)							,				[
									        [
									            {"""score""": 0.9_9_9, """label""": """Sound of a dog"""},
									            {"""score""": 0.0_0_1, """label""": """Sound of vaccum cleaner"""},
									        ],
									    ]
									    * 5							,				)


							@unittest.skip("""No models are available in TF"""		)
							def      _snake_case  (							self    :str		)   ->   Optional[int]:



									"""simple docstring"""



									pass