

'''simple docstring'''


import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    WavaVecaConformerConfig,
    WavaVecaConformerForCTC,
    WavaVecaConformerForPreTraining,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
lowerCAmelCase__										= logging.get_logger(__name__)

lowerCAmelCase__										= {
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.linear_k": "encoder.layers.*.self_attn.linear_k",
    "self_attn.linear_v": "encoder.layers.*.self_attn.linear_v",
    "self_attn.linear_q": "encoder.layers.*.self_attn.linear_q",
    "self_attn.pos_bias_u": "encoder.layers.*.self_attn.pos_bias_u",
    "self_attn.pos_bias_v": "encoder.layers.*.self_attn.pos_bias_v",
    "self_attn.linear_out": "encoder.layers.*.self_attn.linear_out",
    "self_attn.linear_pos": "encoder.layers.*.self_attn.linear_pos",
    "self_attn.rotary_emb": "encoder.embed_positions",
    "self_attn_layer_norm": "encoder.layers.*.self_attn_layer_norm",
    "conv_module.pointwise_conv1": "encoder.layers.*.conv_module.pointwise_conv1",
    "conv_module.pointwise_conv2": "encoder.layers.*.conv_module.pointwise_conv2",
    "conv_module.depthwise_conv": "encoder.layers.*.conv_module.depthwise_conv",
    "conv_module.batch_norm": "encoder.layers.*.conv_module.batch_norm",
    "conv_module.layer_norm": "encoder.layers.*.conv_module.layer_norm",
    "ffn1.w_1": "encoder.layers.*.ffn1.intermediate_dense",
    "ffn1.w_2": "encoder.layers.*.ffn1.output_dense",
    "ffn1.layer_norm": "encoder.layers.*.ffn1_layer_norm",
    "ffn2.w_1": "encoder.layers.*.ffn2.intermediate_dense",
    "ffn2.w_2": "encoder.layers.*.ffn2.output_dense",
    "ffn2.layer_norm": "encoder.layers.*.ffn2_layer_norm",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "feature_projection.layer_norm",
    "quantizer.weight_proj": "quantizer.weight_proj",
    "quantizer.vars": "quantizer.codevectors",
    "project_q": "project_q",
    "final_proj": "project_hid",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}
lowerCAmelCase__										= [
    "lm_head",
    "quantizer.weight_proj",
    "quantizer.codevectors",
    "project_q",
    "project_hid",
]







def 						SCREAMING_SNAKE_CASE(  UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase  )    ->    int:
	for attribute in key.split('.'  ):
		UpperCAmelCase_		:      Dict       	=					getattr(UpperCamelCase   ,UpperCamelCase  )

	if weight_type is not None:
		UpperCAmelCase_		:      Optional[Any]       	=					getattr(UpperCamelCase   ,UpperCamelCase  ).shape
	else:
		UpperCAmelCase_		:      Union[str, Any]       	=					hf_pointer.shape

	if hf_shape != value.shape:
		raise ValueError(
		    f"""Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be"""
		    f""" {value.shape} for {full_name}"""  )

	if weight_type == "weight":
		UpperCAmelCase_		:      List[str]       	=					value
	elif weight_type == "weight_g":
		UpperCAmelCase_		:      Optional[int]       	=					value
	elif weight_type == "weight_v":
		UpperCAmelCase_		:      List[Any]       	=					value
	elif weight_type == "bias":
		UpperCAmelCase_		:      List[str]       	=					value
	elif weight_type == "running_mean":
		UpperCAmelCase_		:      List[Any]       	=					value
	elif weight_type == "running_var":
		UpperCAmelCase_		:      Optional[int]       	=					value
	elif weight_type == "num_batches_tracked":
		UpperCAmelCase_		:      List[Any]       	=					value
	elif weight_type == "inv_freq":
		UpperCAmelCase_		:      Dict       	=					value
	else:
		UpperCAmelCase_		:      Tuple       	=					value

	logger.info(f"""{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}."""  )







def 						SCREAMING_SNAKE_CASE(  UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase  )    ->    Tuple:
	UpperCAmelCase_		:      int       	=					[]
	UpperCAmelCase_		:      Union[str, Any]       	=					fairseq_model.state_dict()

	UpperCAmelCase_		:      Dict       	=					hf_model.wavaveca_conformer.feature_extractor

	for name, value in fairseq_dict.items():
		UpperCAmelCase_		:      Dict       	=					False
		if "conv_layers" in name:
			load_conv_layer(
			    UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,hf_model.config.feat_extract_norm == 'group'   ,)
			UpperCAmelCase_		:      Tuple       	=					True
		else:
			for key, mapped_key in MAPPING.items():
				UpperCAmelCase_		:      List[str]       	=					'wav2vec2_conformer.' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
				if key in name or key.split('w2v_model.'  )[-1] == name.split('.'  )[0]:
					UpperCAmelCase_		:      str       	=					True
					if "*" in mapped_key:
						UpperCAmelCase_		:      Optional[int]       	=					name.split(UpperCamelCase  )[0].split('.'  )[-2]
						UpperCAmelCase_		:      Union[str, Any]       	=					mapped_key.replace('*'   ,UpperCamelCase  )
					if "pos_bias_u" in name:
						UpperCAmelCase_		:      List[Any]       	=					None
					elif "pos_bias_v" in name:
						UpperCAmelCase_		:      int       	=					None
					elif "weight_g" in name:
						UpperCAmelCase_		:      Tuple       	=					'weight_g'
					elif "weight_v" in name:
						UpperCAmelCase_		:      List[Any]       	=					'weight_v'
					elif "bias" in name:
						UpperCAmelCase_		:      int       	=					'bias'
					elif "weight" in name:
						# TODO: don't match quantizer.weight_proj
						UpperCAmelCase_		:      Dict       	=					'weight'
					elif "running_mean" in name:
						UpperCAmelCase_		:      int       	=					'running_mean'
					elif "inv_freq" in name:
						UpperCAmelCase_		:      Any       	=					'inv_freq'
					elif "running_var" in name:
						UpperCAmelCase_		:      Optional[Any]       	=					'running_var'
					elif "num_batches_tracked" in name:
						UpperCAmelCase_		:      Union[str, Any]       	=					'num_batches_tracked'
					else:
						UpperCAmelCase_		:      Union[str, Any]       	=					None
					set_recursively(UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase  )
				continue
		if not is_used:
			unused_weights.append(UpperCamelCase  )

	logger.warning(f"""Unused weights: {unused_weights}"""  )







def 						SCREAMING_SNAKE_CASE(  UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase  )    ->    Optional[Any]:
	UpperCAmelCase_		:      List[Any]       	=					full_name.split('conv_layers.'  )[-1]
	UpperCAmelCase_		:      Dict       	=					name.split('.'  )
	UpperCAmelCase_		:      Tuple       	=					int(items[0]  )
	UpperCAmelCase_		:      Tuple       	=					int(items[1]  )

	if type_id == 0:
		if "bias" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
				raise ValueError(
				    f"""{full_name} has size {value.shape}, but"""
				    f""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""  )
			UpperCAmelCase_		:      Optional[int]       	=					value
			logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}."""  )
		elif "weight" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
				raise ValueError(
				    f"""{full_name} has size {value.shape}, but"""
				    f""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""  )
			UpperCAmelCase_		:      Optional[Any]       	=					value
			logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}."""  )
	elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
		if "bias" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
				raise ValueError(
				    f"""{full_name} has size {value.shape}, but"""
				    f""" {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found."""  )
			UpperCAmelCase_		:      int       	=					value
			logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""  )
		elif "weight" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
				raise ValueError(
				    f"""{full_name} has size {value.shape}, but"""
				    f""" {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found."""  )
			UpperCAmelCase_		:      int       	=					value
			logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""  )
	else:
		unused_weights.append(UpperCamelCase  )







@torch.no_grad()
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase=None   ,UpperCamelCase=None   ,UpperCamelCase=True  )    ->    List[str]:
	if config_path is not None:
		UpperCAmelCase_		:      List[str]       	=					WavaVecaConformerConfig.from_pretrained(UpperCamelCase   ,hidden_act='swish'  )
	else:
		UpperCAmelCase_		:      Optional[int]       	=					WavaVecaConformerConfig()

	if "rope" in checkpoint_path:
		UpperCAmelCase_		:      Tuple       	=					'rotary'

	if is_finetuned:
		if dict_path:
			UpperCAmelCase_		:      Optional[int]       	=					Dictionary.load(UpperCamelCase  )

			# important change bos & pad token id since CTC symbol is <pad> and
			# not <s> as in fairseq
			UpperCAmelCase_		:      int       	=					target_dict.pad_index
			UpperCAmelCase_		:      str       	=					target_dict.bos_index
			UpperCAmelCase_		:      Any       	=					target_dict.eos_index
			UpperCAmelCase_		:      List[Any]       	=					len(target_dict.symbols  )
			UpperCAmelCase_		:      str       	=					os.path.join(UpperCamelCase   ,'vocab.json'  )
			if not os.path.isdir(UpperCamelCase  ):
				logger.error('--pytorch_dump_folder_path ({}) should be a directory'.format(UpperCamelCase  )  )
				return
			os.makedirs(UpperCamelCase   ,exist_ok=UpperCamelCase  )
			UpperCAmelCase_		:      int       	=					target_dict.indices

			# fairseq has the <pad> and <s> switched
			UpperCAmelCase_		:      List[Any]       	=					0
			UpperCAmelCase_		:      Optional[Any]       	=					1
			with open(UpperCamelCase   ,'w'   ,encoding='utf-8'  ) as vocab_handle:
				json.dump(UpperCamelCase   ,UpperCamelCase  )
			UpperCAmelCase_		:      Optional[Any]       	=					WavaVecaCTCTokenizer(
			    UpperCamelCase   ,unk_token=target_dict.unk_word   ,pad_token=target_dict.pad_word   ,bos_token=target_dict.bos_word   ,eos_token=target_dict.eos_word   ,word_delimiter_token='|'   ,do_lower_case=UpperCamelCase   ,)
			UpperCAmelCase_		:      Any       	=					True if config.feat_extract_norm == 'layer' else False
			UpperCAmelCase_		:      Dict       	=					WavaVecaFeatureExtractor(
			    feature_size=1   ,sampling_rate=1_6_0_0_0   ,padding_value=0   ,do_normalize=UpperCamelCase   ,return_attention_mask=UpperCamelCase   ,)
			UpperCAmelCase_		:      str       	=					WavaVecaProcessor(feature_extractor=UpperCamelCase   ,tokenizer=UpperCamelCase  )
			processor.save_pretrained(UpperCamelCase  )

		UpperCAmelCase_		:      List[str]       	=					WavaVecaConformerForCTC(UpperCamelCase  )
	else:
		UpperCAmelCase_		:      str       	=					WavaVecaConformerForPreTraining(UpperCamelCase  )

	if is_finetuned:
		UpperCAmelCase_		:      Union[str, Any]       	=					fairseq.checkpoint_utils.load_model_ensemble_and_task(
		    [checkpoint_path]   ,arg_overrides={'data': '/'.join(dict_path.split('/'  )[:-1]  )}  )
	else:
		UpperCAmelCase_		:      Dict       	=					argparse.Namespace(task='audio_pretraining'  )
		UpperCAmelCase_		:      Any       	=					fairseq.tasks.setup_task(UpperCamelCase  )

		UpperCAmelCase_		:      Tuple       	=					fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]   ,task=UpperCamelCase  )

	UpperCAmelCase_		:      Optional[Any]       	=					model[0].eval()

	recursively_load_weights(UpperCamelCase   ,UpperCamelCase   ,not is_finetuned  )

	hf_wavavec.save_pretrained(UpperCamelCase  )


if __name__ == "__main__":
						lowerCAmelCase__										= argparse.ArgumentParser()
						parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
						parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint")
						parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model")
						parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
						parser.add_argument(
						    "--not_finetuned", action="store_true", help="Whether the model to convert is a fine-tuned model or not"
						)
						lowerCAmelCase__										= parser.parse_args()
						convert_wavaveca_conformer_checkpoint(
						    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
						)









'''simple docstring'''


import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class 					lowercase						:


							def __init__(			self    ,			_snake_case    ,			_snake_case=13    ,			_snake_case=7    ,			_snake_case=True    ,			_snake_case=True    ,			_snake_case=False    ,			_snake_case=True    ,			_snake_case=99    ,			_snake_case=32    ,			_snake_case=5    ,			_snake_case=4    ,			_snake_case=37    ,			_snake_case="gelu"    ,			_snake_case=0.1    ,			_snake_case=0.1    ,			_snake_case=512    ,			_snake_case=16    ,			_snake_case=2    ,			_snake_case=0.02    ,			_snake_case=3    ,			_snake_case=4    ,			_snake_case=None    ,			)							->				str:
								UpperCAmelCase_		:      Dict       	=					parent
								UpperCAmelCase_		:      Tuple       	=					batch_size
								UpperCAmelCase_		:      Tuple       	=					seq_length
								UpperCAmelCase_		:      Dict       	=					is_training
								UpperCAmelCase_		:      Tuple       	=					use_input_mask
								UpperCAmelCase_		:      List[Any]       	=					use_token_type_ids
								UpperCAmelCase_		:      Optional[int]       	=					use_labels
								UpperCAmelCase_		:      List[str]       	=					vocab_size
								UpperCAmelCase_		:      int       	=					hidden_size
								UpperCAmelCase_		:      Any       	=					num_hidden_layers
								UpperCAmelCase_		:      Optional[int]       	=					num_attention_heads
								UpperCAmelCase_		:      List[str]       	=					intermediate_size
								UpperCAmelCase_		:      Union[str, Any]       	=					hidden_act
								UpperCAmelCase_		:      Optional[int]       	=					hidden_dropout_prob
								UpperCAmelCase_		:      Any       	=					attention_probs_dropout_prob
								UpperCAmelCase_		:      Dict       	=					max_position_embeddings
								UpperCAmelCase_		:      Union[str, Any]       	=					type_vocab_size
								UpperCAmelCase_		:      int       	=					type_sequence_label_size
								UpperCAmelCase_		:      List[str]       	=					initializer_range
								UpperCAmelCase_		:      Any       	=					num_labels
								UpperCAmelCase_		:      Optional[Any]       	=					num_choices
								UpperCAmelCase_		:      str       	=					scope


							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.vocab_size)

								UpperCAmelCase_		:      Any       	=					None
								if self.use_input_mask:
									UpperCAmelCase_		:      Optional[Any]       	=					random_attention_mask([self.batch_size, self.seq_length])

								UpperCAmelCase_		:      Union[str, Any]       	=					None
								if self.use_token_type_ids:
									UpperCAmelCase_		:      Tuple       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.type_vocab_size)

								UpperCAmelCase_		:      Dict       	=					None
								UpperCAmelCase_		:      List[str]       	=					None
								UpperCAmelCase_		:      Optional[Any]       	=					None
								if self.use_labels:
									UpperCAmelCase_		:      Any       	=					ids_tensor([self.batch_size]    ,			self.type_sequence_label_size)
									UpperCAmelCase_		:      List[str]       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.num_labels)
									UpperCAmelCase_		:      Tuple       	=					ids_tensor([self.batch_size]    ,			self.num_choices)

								UpperCAmelCase_		:      Tuple       	=					self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def 						_snake_case							(			self)							->				Optional[Any]:
								return OpenLlamaConfig(
								    vocab_size=self.vocab_size    ,			hidden_size=self.hidden_size    ,			num_hidden_layers=self.num_hidden_layers    ,			num_attention_heads=self.num_attention_heads    ,			intermediate_size=self.intermediate_size    ,			hidden_act=self.hidden_act    ,			hidden_dropout_prob=self.hidden_dropout_prob    ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,			max_position_embeddings=self.max_position_embeddings    ,			type_vocab_size=self.type_vocab_size    ,			is_decoder=_snake_case    ,			initializer_range=self.initializer_range    ,			use_stable_embedding=_snake_case    ,			)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case)							->				Union[str, Any]:
								UpperCAmelCase_		:      str       	=					OpenLlamaModel(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Dict       	=					model(_snake_case    ,			attention_mask=_snake_case)
								UpperCAmelCase_		:      List[str]       	=					model(_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				int:
								UpperCAmelCase_		:      Union[str, Any]       	=					True
								UpperCAmelCase_		:      List[Any]       	=					OpenLlamaModel(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Optional[int]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			)
								UpperCAmelCase_		:      Any       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			)
								UpperCAmelCase_		:      Optional[int]       	=					model(_snake_case    ,			attention_mask=_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				Any:
								UpperCAmelCase_		:      Optional[int]       	=					OpenLlamaForCausalLM(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Optional[int]       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.parent.assertEqual(result.logits.shape    ,			(self.batch_size, self.seq_length, self.vocab_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				Union[str, Any]:
								UpperCAmelCase_		:      Any       	=					True
								UpperCAmelCase_		:      Optional[Any]       	=					True
								UpperCAmelCase_		:      int       	=					OpenLlamaForCausalLM(config=_snake_case)
								model.to(_snake_case)
								model.eval()

								# first forward pass
								UpperCAmelCase_		:      List[str]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			use_cache=_snake_case    ,			)
								UpperCAmelCase_		:      Dict       	=					outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								UpperCAmelCase_		:      str       	=					ids_tensor((self.batch_size, 3)    ,			config.vocab_size)
								UpperCAmelCase_		:      int       	=					ids_tensor((self.batch_size, 3)    ,			vocab_size=2)

								# append to next input_ids and
								UpperCAmelCase_		:      Tuple       	=					torch.cat([input_ids, next_tokens]    ,			dim=-1)
								UpperCAmelCase_		:      List[str]       	=					torch.cat([input_mask, next_mask]    ,			dim=-1)

								UpperCAmelCase_		:      Optional[Any]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			output_hidden_states=_snake_case    ,			)['hidden_states'][0]
								UpperCAmelCase_		:      List[str]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			past_key_values=_snake_case    ,			output_hidden_states=_snake_case    ,			)['hidden_states'][0]

								# select random slice
								UpperCAmelCase_		:      Optional[Any]       	=					ids_tensor((1,)    ,			output_from_past.shape[-1]).item()
								UpperCAmelCase_		:      Optional[int]       	=					output_from_no_past[:, -3:, random_slice_idx].detach()
								UpperCAmelCase_		:      Union[str, Any]       	=					output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-3))





							def 						_snake_case							(			self)							->				List[Any]:
								UpperCAmelCase_		:      Optional[Any]       	=					self.prepare_config_and_inputs()
								(
								    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    
								)		:      Union[str, Any]       	=					config_and_inputs
								UpperCAmelCase_		:      List[str]       	=					{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict



@require_torch
class 					lowercase						(  a_,					a_,					a_,					unittest.TestCase ):
							_lowerCamelCase		:      Dict=   (
							    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
							)
							_lowerCamelCase		:      str=   (OpenLlamaForCausalLM,) if is_torch_available() else ()
							_lowerCamelCase		:      Optional[int]=   (
							    {
							        "feature-extraction": OpenLlamaModel,
							        "text-classification": OpenLlamaForSequenceClassification,
							        "text-generation": OpenLlamaForCausalLM,
							        "zero-shot": OpenLlamaForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							_lowerCamelCase		:      Tuple=   False
							_lowerCamelCase		:      int=   False


							def 						_snake_case							(			self)							->				int:
								UpperCAmelCase_		:      Any       	=					OpenLlamaModelTester(self)
								UpperCAmelCase_		:      Union[str, Any]       	=					ConfigTester(self    ,			config_class=_snake_case    ,			hidden_size=37)


							def 						_snake_case							(			self)							->				Optional[Any]:
								self.config_tester.run_common_tests()


							def 						_snake_case							(			self)							->				str:
								UpperCAmelCase_		:      Tuple       	=					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_snake_case)


							def 						_snake_case							(			self)							->				Union[str, Any]:
								UpperCAmelCase_		:      str       	=					self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
									UpperCAmelCase_		:      List[Any]       	=					type
									self.model_tester.create_and_check_model(*_snake_case)


							def 						_snake_case							(			self)							->				Union[str, Any]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      List[Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      List[str]       	=					3
								UpperCAmelCase_		:      List[str]       	=					input_dict['input_ids']
								UpperCAmelCase_		:      Optional[Any]       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.model_tester.batch_size]    ,			self.model_tester.type_sequence_label_size)
								UpperCAmelCase_		:      List[Any]       	=					OpenLlamaForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Tuple       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							def 						_snake_case							(			self)							->				Any:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Dict       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      Optional[Any]       	=					3
								UpperCAmelCase_		:      int       	=					'single_label_classification'
								UpperCAmelCase_		:      int       	=					input_dict['input_ids']
								UpperCAmelCase_		:      Any       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.model_tester.batch_size]    ,			self.model_tester.type_sequence_label_size)
								UpperCAmelCase_		:      int       	=					OpenLlamaForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      List[str]       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							def 						_snake_case							(			self)							->				Optional[int]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      List[Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      int       	=					3
								UpperCAmelCase_		:      Any       	=					'multi_label_classification'
								UpperCAmelCase_		:      Tuple       	=					input_dict['input_ids']
								UpperCAmelCase_		:      str       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Any       	=					ids_tensor(
								    [self.model_tester.batch_size, config.num_labels]    ,			self.model_tester.type_sequence_label_size).to(torch.float)
								UpperCAmelCase_		:      Optional[Any]       	=					OpenLlamaForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      List[Any]       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test')
							def 						_snake_case							(			self)							->				Optional[int]:
								pass





							@parameterized.expand([('linear',), ('dynamic',)])
							def 						_snake_case							(			self    ,			_snake_case)							->				List[str]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Any       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      List[Any]       	=					ids_tensor([1, 10]    ,			config.vocab_size)
								UpperCAmelCase_		:      List[str]       	=					ids_tensor([1, int(config.max_position_embeddings * 1.5)]    ,			config.vocab_size)

								set_seed(42)  # Fixed seed at init time so the two models get the same random weights
								UpperCAmelCase_		:      int       	=					OpenLlamaModel(_snake_case)
								original_model.to(_snake_case)
								original_model.eval()
								UpperCAmelCase_		:      Optional[Any]       	=					original_model(_snake_case).last_hidden_state
								UpperCAmelCase_		:      Tuple       	=					original_model(_snake_case).last_hidden_state

								set_seed(42)  # Fixed seed at init time so the two models get the same random weights
								UpperCAmelCase_		:      Dict       	=					{'type': scaling_type, 'factor': 10.0}
								UpperCAmelCase_		:      Union[str, Any]       	=					OpenLlamaModel(_snake_case)
								scaled_model.to(_snake_case)
								scaled_model.eval()
								UpperCAmelCase_		:      Dict       	=					scaled_model(_snake_case).last_hidden_state
								UpperCAmelCase_		:      Optional[int]       	=					scaled_model(_snake_case).last_hidden_state

								# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
								# maximum sequence length, so the outputs for the short input should match.
								if scaling_type == "dynamic":
									self.assertTrue(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-5))
								else:
									self.assertFalse(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-5))

								# The output should be different for long inputs
								self.assertFalse(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-5))






