
"""simple docstring"""


import argparse

import torch

from transformers import (
    SpeechTaConfig,
    SpeechTaFeatureExtractor,
    SpeechTaForSpeechToSpeech,
    SpeechTaForSpeechToText,
    SpeechTaForTextToSpeech,
    SpeechTaProcessor,
    SpeechTaTokenizer,
    logging,
)
from transformers.tokenization_utils import AddedToken


logging.set_verbosity_info()
_lowercase							   =				logging.get_logger('''transformers.models.speecht5''')

_lowercase							   =				{
    '''speech_encoder_prenet.layer_norm''': '''speecht5.encoder.prenet.feature_projection.layer_norm''',
    '''speech_encoder_prenet.post_extract_proj''': '''speecht5.encoder.prenet.feature_projection.projection''',
    '''speech_encoder_prenet.pos_conv.0''': '''speecht5.encoder.prenet.pos_conv_embed.conv''',
    '''speech_encoder_prenet.mask_emb''': '''speecht5.encoder.prenet.masked_spec_embed''',
}
_lowercase							   =				{
    '''text_encoder_prenet.encoder_prenet.0''': '''speecht5.encoder.prenet.embed_tokens''',
    '''text_encoder_prenet.encoder_prenet.1.alpha''': '''speecht5.encoder.prenet.encode_positions.alpha''',
}
_lowercase							   =				{
    '''speech_decoder_prenet.decoder_prenet.0.0.prenet.0.0''': '''speecht5.decoder.prenet.layers.0''',
    '''speech_decoder_prenet.decoder_prenet.0.0.prenet.1.0''': '''speecht5.decoder.prenet.layers.1''',
    '''speech_decoder_prenet.decoder_prenet.0.1''': '''speecht5.decoder.prenet.final_layer''',
    '''speech_decoder_prenet.decoder_prenet.1.alpha''': '''speecht5.decoder.prenet.encode_positions.alpha''',
    '''speech_decoder_prenet.spkembs_layer.0''': '''speecht5.decoder.prenet.speaker_embeds_layer''',
}
_lowercase							   =				{
    '''speech_decoder_postnet.feat_out''': '''speech_decoder_postnet.feat_out''',
    '''speech_decoder_postnet.prob_out''': '''speech_decoder_postnet.prob_out''',
    '''speech_decoder_postnet.postnet.postnet.0.0''': '''speech_decoder_postnet.layers.0.conv''',
    '''speech_decoder_postnet.postnet.postnet.0.1''': '''speech_decoder_postnet.layers.0.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.1.0''': '''speech_decoder_postnet.layers.1.conv''',
    '''speech_decoder_postnet.postnet.postnet.1.1''': '''speech_decoder_postnet.layers.1.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.2.0''': '''speech_decoder_postnet.layers.2.conv''',
    '''speech_decoder_postnet.postnet.postnet.2.1''': '''speech_decoder_postnet.layers.2.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.3.0''': '''speech_decoder_postnet.layers.3.conv''',
    '''speech_decoder_postnet.postnet.postnet.3.1''': '''speech_decoder_postnet.layers.3.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.4.0''': '''speech_decoder_postnet.layers.4.conv''',
    '''speech_decoder_postnet.postnet.postnet.4.1''': '''speech_decoder_postnet.layers.4.batch_norm''',
}
_lowercase							   =				{
    '''text_decoder_prenet.embed_tokens''': '''speecht5.decoder.prenet.embed_tokens''',
}
_lowercase							   =				{
    '''text_decoder_postnet.output_projection''': '''text_decoder_postnet.lm_head''',
}
_lowercase							   =				{
    '''encoder.layers.*.self_attn.k_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.k_proj''',
    '''encoder.layers.*.self_attn.v_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.v_proj''',
    '''encoder.layers.*.self_attn.q_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.q_proj''',
    '''encoder.layers.*.self_attn.out_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.out_proj''',
    '''encoder.layers.*.self_attn_layer_norm''': '''speecht5.encoder.wrapped_encoder.layers.*.layer_norm''',
    '''encoder.layers.*.fc1''': '''speecht5.encoder.wrapped_encoder.layers.*.feed_forward.intermediate_dense''',
    '''encoder.layers.*.fc2''': '''speecht5.encoder.wrapped_encoder.layers.*.feed_forward.output_dense''',
    '''encoder.layers.*.final_layer_norm''': '''speecht5.encoder.wrapped_encoder.layers.*.final_layer_norm''',
    '''encoder.layer_norm''': '''speecht5.encoder.wrapped_encoder.layer_norm''',
    '''encoder.pos_emb.pe_k''': '''speecht5.encoder.wrapped_encoder.embed_positions.pe_k''',
}
_lowercase							   =				{
    '''decoder.layers.*.self_attn.k_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.k_proj''',
    '''decoder.layers.*.self_attn.v_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.v_proj''',
    '''decoder.layers.*.self_attn.q_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.q_proj''',
    '''decoder.layers.*.self_attn.out_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.out_proj''',
    '''decoder.layers.*.self_attn_layer_norm''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn_layer_norm''',
    '''decoder.layers.*.encoder_attn.k_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.k_proj''',
    '''decoder.layers.*.encoder_attn.v_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.v_proj''',
    '''decoder.layers.*.encoder_attn.q_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.q_proj''',
    '''decoder.layers.*.encoder_attn.out_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.out_proj''',
    '''decoder.layers.*.encoder_attn_layer_norm''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn_layer_norm''',
    '''decoder.layers.*.fc1''': '''speecht5.decoder.wrapped_decoder.layers.*.feed_forward.intermediate_dense''',
    '''decoder.layers.*.fc2''': '''speecht5.decoder.wrapped_decoder.layers.*.feed_forward.output_dense''',
    '''decoder.layers.*.final_layer_norm''': '''speecht5.decoder.wrapped_decoder.layers.*.final_layer_norm''',
}
_lowercase							   =				{
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_TEXT_DECODER_PRENET,
    **MAPPING_TEXT_DECODER_POSTNET,
}
_lowercase							   =				{
    **MAPPING_TEXT_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
_lowercase							   =				{
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
_lowercase							   =				[]
_lowercase							   =				[
    '''encoder.version''',
    '''encoder.layers.*.norm_k.weight''',
    '''encoder.layers.*.norm_k.bias''',
    '''decoder.version''',
    '''decoder.layers.*.norm_k.weight''',
    '''decoder.layers.*.norm_k.bias''',
    '''decoder.pos_emb.pe_k''',
    '''speech_encoder_prenet.embed_positions._float_tensor''',
    '''text_decoder_prenet.embed_positions._float_tensor''',
]
_lowercase							   =				IGNORE_KEYS + [
    '''encoder.proj''',
    '''text_encoder_prenet.*''',
    '''speech_decoder_prenet.*''',
    '''speech_decoder_postnet.*''',
]
_lowercase							   =				IGNORE_KEYS + [
    '''encoder.proj''',
    '''speech_encoder_prenet.*''',
    '''text_decoder_prenet.*''',
    '''text_decoder_postnet.*''',
]
_lowercase							   =				IGNORE_KEYS + [
    '''encoder.proj''',
    '''text_encoder_prenet.*''',
    '''text_decoder_prenet.*''',
    '''text_decoder_postnet.*''',
]






def       _snake_case				(		snake_case__   :     str	,      snake_case__   :     Dict	,      snake_case__   :     Dict	,      snake_case__   :     Optional[Any]	,      snake_case__   :     Any     ):
	for attribute in key.split('.'     ):
		A						   =   getattr(snake_case__	,      snake_case__     )

	if weight_type is not None:
		A						   =   getattr(snake_case__	,      snake_case__     ).shape
	else:
		A						   =   hf_pointer.shape

	if hf_shape != value.shape:
		raise ValueError(
		    F'Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'
		    F' {value.shape} for {full_name}'     )

	if weight_type == "weight":
		A						   =   value
	elif weight_type == "weight_g":
		A						   =   value
	elif weight_type == "weight_v":
		A						   =   value
	elif weight_type == "bias":
		A						   =   value
	elif weight_type == "running_mean":
		A						   =   value
	elif weight_type == "running_var":
		A						   =   value
	elif weight_type == "num_batches_tracked":
		A						   =   value
	else:
		A						   =   value

	logger.info(F'{key + ("." + weight_type if weight_type is not None else "")} was initialized from {full_name}.'     )






def       _snake_case				(		snake_case__   :     int	,      snake_case__   :     str     ):
	for key in ignore_keys:
		if key.endswith('.*'     ):
			if name.startswith(key[:-1]     ):
				return True
		elif ".*." in key:
			A   ,      A						   =   key.split('.*.'     )
			if prefix in name and suffix in name:
				return True
		elif key in name:
			return True
	return False






def       _snake_case				(		snake_case__   :     Optional[int]	,      snake_case__   :     List[str]	,      snake_case__   :     Union[str, Any]     ):
	A						   =   []

	if task == "s2t":
		A						   =   hf_model.speechta.encoder.prenet.feature_encoder
		A						   =   MAPPING_S2T
		A						   =   IGNORE_KEYS_S2T
	elif task == "t2s":
		A						   =   None
		A						   =   MAPPING_T2S
		A						   =   IGNORE_KEYS_T2S
	elif task == "s2s":
		A						   =   hf_model.speechta.encoder.prenet.feature_encoder
		A						   =   MAPPING_S2S
		A						   =   IGNORE_KEYS_S2S
	else:
		raise ValueError(F'Unsupported task: {task}'     )

	for name, value in fairseq_dict.items():
		if should_ignore(snake_case__	,      snake_case__     ):
			logger.info(F'{name} was ignored'     )
			continue

		A						   =   False
		if "conv_layers" in name:
			load_conv_layer(
			    snake_case__	,      snake_case__	,      snake_case__	,      snake_case__	,      hf_model.config.feat_extract_norm == 'group'	,      )
			A						   =   True
		else:
			for key, mapped_key in MAPPING.items():
				# mapped_key = "speecht5." + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key

				if "*" in key:
					A   ,      A						   =   key.split('.*.'     )
					if prefix in name and suffix in name:
						A						   =   suffix

                # if key in name or key.split("w2v_model.")[-1] == name.split(".")[0]:
				if key in name:
					A						   =   True
					if "*" in mapped_key:
						A						   =   name.split(snake_case__     )[0].split('.'     )[-2]
						A						   =   mapped_key.replace('*'	,      snake_case__     )
					if "weight_g" in name:
						A						   =   'weight_g'
					elif "weight_v" in name:
						A						   =   'weight_v'
					elif "bias" in name:
						A						   =   'bias'
					elif "weight" in name:
						A						   =   'weight'
					elif "running_mean" in name:
						A						   =   'running_mean'
					elif "running_var" in name:
						A						   =   'running_var'
					elif "num_batches_tracked" in name:
						A						   =   'num_batches_tracked'
					else:
						A						   =   None
					set_recursively(snake_case__	,      snake_case__	,      snake_case__	,      snake_case__	,      snake_case__     )
				continue
		if not is_used:
			unused_weights.append(snake_case__     )

	logger.warning(F'Unused weights: {unused_weights}'     )






def       _snake_case				(		snake_case__   :     Optional[int]	,      snake_case__   :     int	,      snake_case__   :     List[str]	,      snake_case__   :     Optional[Any]	,      snake_case__   :     int     ):
	A						   =   full_name.split('conv_layers.'     )[-1]
	A						   =   name.split('.'     )
	A						   =   int(items[0]     )
	A						   =   int(items[1]     )

	if type_id == 0:
		if "bias" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
				raise ValueError(
				    F'{full_name} has size {value.shape}, but'
				    F' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'     )
			A						   =   value
			logger.info(F'Feat extract conv layer {layer_id} was initialized from {full_name}.'     )
		elif "weight" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
				raise ValueError(
				    F'{full_name} has size {value.shape}, but'
				    F' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'     )
			A						   =   value
			logger.info(F'Feat extract conv layer {layer_id} was initialized from {full_name}.'     )
	elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
		if "bias" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
				raise ValueError(
				    F'{full_name} has size {value.shape}, but'
				    F' {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found.'     )
			A						   =   value
			logger.info(F'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'     )
		elif "weight" in name:
			if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
				raise ValueError(
				    F'{full_name} has size {value.shape}, but'
				    F' {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found.'     )
			A						   =   value
			logger.info(F'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'     )
	else:
		unused_weights.append(snake_case__     )






@torch.no_grad()
def       _snake_case				(		snake_case__   :     Union[str, Any]	,      snake_case__   :     Union[str, Any]	,      snake_case__   :     Dict	,      snake_case__   :     int=None	,      snake_case__   :     Tuple=None	,      snake_case__   :     int=None	,      ):
	if config_path is not None:
		A						   =   SpeechTaConfig.from_pretrained(snake_case__     )
	else:
		A						   =   SpeechTaConfig()

	if task == "s2t":
		A						   =   config.max_text_positions
		A						   =   SpeechTaForSpeechToText(snake_case__     )
	elif task == "t2s":
		A						   =   1876
		A						   =   600
		A						   =   config.max_speech_positions
		A						   =   SpeechTaForTextToSpeech(snake_case__     )
	elif task == "s2s":
		A						   =   1876
		A						   =   config.max_speech_positions
		A						   =   SpeechTaForSpeechToSpeech(snake_case__     )
	else:
		raise ValueError(F'Unknown task name: {task}'     )

	if vocab_path:
		A						   =   SpeechTaTokenizer(snake_case__	,      model_max_length=config.max_text_positions     )

		# Mask token behaves like a normal word, i.e. include the space before it
		A						   =   AddedToken('<mask>'	,      lstrip=snake_case__	,      rstrip=snake_case__     )
		A						   =   mask_token
		tokenizer.add_special_tokens({'mask_token': mask_token}     )
		tokenizer.add_tokens(['<ctc_blank>']     )

	A						   =   SpeechTaFeatureExtractor()
	A						   =   SpeechTaProcessor(tokenizer=snake_case__	,      feature_extractor=snake_case__     )
	processor.save_pretrained(snake_case__     )

	A						   =   torch.load(snake_case__     )
	recursively_load_weights(fairseq_checkpoint['model']	,      snake_case__	,      snake_case__     )

	model.save_pretrained(snake_case__     )

	if repo_id:
		print('Pushing to the hub...'     )
		processor.push_to_hub(snake_case__     )
		model.push_to_hub(snake_case__     )


if __name__ == "__main__":
		_lowercase							   =				argparse.ArgumentParser()
		parser.add_argument(
		    '''--task''',
		    default='''s2t''',
		    type=str,
		    help='''Type of the SpeechT5 model you\'d like to convert. Should be one of \'s2t\', \'t2s\', \'s2s\'.''',
		)
		parser.add_argument('''--checkpoint_path''', required=True, default=None, type=str, help='''Path to fairseq checkpoint''')
		parser.add_argument('''--vocab_path''', default=None, type=str, help='''Path to SentencePiece model''')
		parser.add_argument('''--config_path''', default=None, type=str, help='''Path to hf config.json of model to convert''')
		parser.add_argument(
		    '''--pytorch_dump_folder_path''', required=True, default=None, type=str, help='''Path to the output PyTorch model.'''
		)
		parser.add_argument(
		    '''--push_to_hub''', default=None, type=str, help='''Where to upload the converted model on the 🤗 hub.'''
		)

		_lowercase							   =				parser.parse_args()
		convert_speechta_checkpoint(
		    args.task,
		    args.checkpoint_path,
		    args.pytorch_dump_folder_path,
		    args.config_path,
		    args.vocab_path,
		    args.push_to_hub,
		)

"""simple docstring"""


import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    XLMForMultipleChoice,
		    XLMForQuestionAnswering,
		    XLMForQuestionAnsweringSimple,
		    XLMForSequenceClassification,
		    XLMForTokenClassification,
		    XLMModel,
		    XLMWithLMHeadModel,
		)
		from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST







class 				lowerCAmelCase_				:



						'''simple docstring'''


						def __init__( self     :      Any  ,A_     :      Optional[Any]  ,A_     :      Tuple=13  ,A_     :      Optional[Any]=7  ,A_     :      Dict=True  ,A_     :      Optional[Any]=True  ,A_     :      str=True  ,A_     :      Union[str, Any]=True  ,A_     :      Optional[Any]=True  ,A_     :      Tuple=False  ,A_     :      Optional[int]=False  ,A_     :      str=False  ,A_     :      int=2  ,A_     :      Union[str, Any]=99  ,A_     :      int=0  ,A_     :      Dict=32  ,A_     :      List[str]=5  ,A_     :      Any=4  ,A_     :      str=0.1  ,A_     :      Any=0.1  ,A_     :      int=512  ,A_     :      List[Any]=2  ,A_     :      Union[str, Any]=0.02  ,A_     :      Optional[Any]=2  ,A_     :      List[str]=4  ,A_     :      Optional[int]="last"  ,A_     :      str=True  ,A_     :      List[str]=None  ,A_     :      List[Any]=0  ,)     ->   int:
							A						   =   parent
							A						   =   batch_size
							A						   =   seq_length
							A						   =   is_training
							A						   =   use_input_lengths
							A						   =   use_token_type_ids
							A						   =   use_labels
							A						   =   gelu_activation
							A						   =   sinusoidal_embeddings
							A						   =   causal
							A						   =   asm
							A						   =   n_langs
							A						   =   vocab_size
							A						   =   n_special
							A						   =   hidden_size
							A						   =   num_hidden_layers
							A						   =   num_attention_heads
							A						   =   hidden_dropout_prob
							A						   =   attention_probs_dropout_prob
							A						   =   max_position_embeddings
							A						   =   type_sequence_label_size
							A						   =   initializer_range
							A						   =   num_labels
							A						   =   num_choices
							A						   =   summary_type
							A						   =   use_proj
							A						   =   scope
							A						   =   bos_token_id


						def 						_SCREAMING_SNAKE_CASE      ( self     :      List[str]      )     ->   Dict:
							A						   =   ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size      )
							A						   =   random_attention_mask([self.batch_size, self.seq_length]      )

							A						   =   None
							if self.use_input_lengths:
								A						   =   (
								    ids_tensor([self.batch_size]  ,vocab_size=2      ) + self.seq_length - 2
								)  # small variation of seq_length

							A						   =   None
							if self.use_token_type_ids:
								A						   =   ids_tensor([self.batch_size, self.seq_length]  ,self.n_langs      )

							A						   =   None
							A						   =   None
							A						   =   None
							if self.use_labels:
								A						   =   ids_tensor([self.batch_size]  ,self.type_sequence_label_size      )
								A						   =   ids_tensor([self.batch_size, self.seq_length]  ,self.num_labels      )
								A						   =   ids_tensor([self.batch_size]  ,2      ).float()
								A						   =   ids_tensor([self.batch_size]  ,self.num_choices      )

							A						   =   self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_lengths,
							    sequence_labels,
							    token_labels,
							    is_impossible_labels,
							    choice_labels,
							    input_mask,
							)


						def 						_SCREAMING_SNAKE_CASE      ( self     :      List[Any]      )     ->   Any:
							return XLMConfig(
							    vocab_size=self.vocab_size  ,n_special=self.n_special  ,emb_dim=self.hidden_size  ,n_layers=self.num_hidden_layers  ,n_heads=self.num_attention_heads  ,dropout=self.hidden_dropout_prob  ,attention_dropout=self.attention_probs_dropout_prob  ,gelu_activation=self.gelu_activation  ,sinusoidal_embeddings=self.sinusoidal_embeddings  ,asm=self.asm  ,causal=self.causal  ,n_langs=self.n_langs  ,max_position_embeddings=self.max_position_embeddings  ,initializer_range=self.initializer_range  ,summary_type=self.summary_type  ,use_proj=self.use_proj  ,num_labels=self.num_labels  ,bos_token_id=self.bos_token_id  ,)


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Optional[int]  ,A_     :      Optional[Any]  ,A_     :      List[str]  ,A_     :      int  ,A_     :      Dict  ,A_     :      Optional[Any]  ,A_     :      Optional[Any]  ,A_     :      Any  ,A_     :      List[str]  ,A_     :      Optional[int]  ,)     ->   Tuple:
							A						   =   XLMModel(config=A_      )
							model.to(A_      )
							model.eval()
							A						   =   model(A_  ,lengths=A_  ,langs=A_      )
							A						   =   model(A_  ,langs=A_      )
							A						   =   model(A_      )
							self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      int  ,A_     :      str  ,A_     :      Union[str, Any]  ,A_     :      Any  ,A_     :      Any  ,A_     :      Any  ,A_     :      Union[str, Any]  ,A_     :      List[str]  ,A_     :      List[str]  ,A_     :      List[str]  ,)     ->   Union[str, Any]:
							A						   =   XLMWithLMHeadModel(A_      )
							model.to(A_      )
							model.eval()

							A						   =   model(A_  ,token_type_ids=A_  ,labels=A_      )
							self.parent.assertEqual(result.loss.shape  ,()      )
							self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Union[str, Any]  ,A_     :      Tuple  ,A_     :      Tuple  ,A_     :      str  ,A_     :      int  ,A_     :      str  ,A_     :      Optional[Any]  ,A_     :      Any  ,A_     :      Any  ,A_     :      Dict  ,)     ->   List[str]:
							A						   =   XLMForQuestionAnsweringSimple(A_      )
							model.to(A_      )
							model.eval()

							A						   =   model(A_      )

							A						   =   model(A_  ,start_positions=A_  ,end_positions=A_      )
							A						   =   outputs
							self.parent.assertEqual(result.start_logits.shape  ,(self.batch_size, self.seq_length)      )
							self.parent.assertEqual(result.end_logits.shape  ,(self.batch_size, self.seq_length)      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Any  ,A_     :      Optional[int]  ,A_     :      Dict  ,A_     :      Optional[Any]  ,A_     :      List[Any]  ,A_     :      List[str]  ,A_     :      List[Any]  ,A_     :      Optional[Any]  ,A_     :      str  ,A_     :      Any  ,)     ->   Optional[int]:
							A						   =   XLMForQuestionAnswering(A_      )
							model.to(A_      )
							model.eval()

							A						   =   model(A_      )

							A						   =   model(
							    A_  ,start_positions=A_  ,end_positions=A_  ,cls_index=A_  ,is_impossible=A_  ,p_mask=A_  ,)

							A						   =   model(
							    A_  ,start_positions=A_  ,end_positions=A_  ,cls_index=A_  ,is_impossible=A_  ,)

							((A)   ,      )						   =   result_with_labels.to_tuple()

							A						   =   model(A_  ,start_positions=A_  ,end_positions=A_      )

							((A)   ,      )						   =   result_with_labels.to_tuple()

							self.parent.assertEqual(result_with_labels.loss.shape  ,()      )
							self.parent.assertEqual(result.start_top_log_probs.shape  ,(self.batch_size, model.config.start_n_top)      )
							self.parent.assertEqual(result.start_top_index.shape  ,(self.batch_size, model.config.start_n_top)      )
							self.parent.assertEqual(
							    result.end_top_log_probs.shape  ,(self.batch_size, model.config.start_n_top * model.config.end_n_top)      )
							self.parent.assertEqual(
							    result.end_top_index.shape  ,(self.batch_size, model.config.start_n_top * model.config.end_n_top)      )
							self.parent.assertEqual(result.cls_logits.shape  ,(self.batch_size,)      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      int  ,A_     :      Union[str, Any]  ,A_     :      Tuple  ,A_     :      int  ,A_     :      Union[str, Any]  ,A_     :      List[Any]  ,A_     :      Optional[Any]  ,A_     :      Tuple  ,A_     :      Union[str, Any]  ,A_     :      str  ,)     ->   List[Any]:
							A						   =   XLMForSequenceClassification(A_      )
							model.to(A_      )
							model.eval()

							A						   =   model(A_      )
							A						   =   model(A_  ,labels=A_      )
							self.parent.assertEqual(result.loss.shape  ,()      )
							self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.type_sequence_label_size)      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Optional[int]  ,A_     :      Optional[Any]  ,A_     :      str  ,A_     :      Tuple  ,A_     :      List[str]  ,A_     :      Dict  ,A_     :      Dict  ,A_     :      Union[str, Any]  ,A_     :      Dict  ,A_     :      Any  ,)     ->   Any:
							A						   =   self.num_labels
							A						   =   XLMForTokenClassification(A_      )
							model.to(A_      )
							model.eval()

							A						   =   model(A_  ,attention_mask=A_  ,labels=A_      )
							self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.num_labels)      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Optional[int]  ,A_     :      int  ,A_     :      str  ,A_     :      int  ,A_     :      Tuple  ,A_     :      List[Any]  ,A_     :      List[str]  ,A_     :      List[str]  ,A_     :      Optional[Any]  ,A_     :      int  ,)     ->   Tuple:
							A						   =   self.num_choices
							A						   =   XLMForMultipleChoice(config=A_      )
							model.to(A_      )
							model.eval()
							A						   =   input_ids.unsqueeze(1      ).expand(-1  ,self.num_choices  ,-1      ).contiguous()
							A						   =   token_type_ids.unsqueeze(1      ).expand(-1  ,self.num_choices  ,-1      ).contiguous()
							A						   =   input_mask.unsqueeze(1      ).expand(-1  ,self.num_choices  ,-1      ).contiguous()
							A						   =   model(
							    A_  ,attention_mask=A_  ,token_type_ids=A_  ,labels=A_  ,)
							self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_choices)      )




						def 						_SCREAMING_SNAKE_CASE      ( self     :      str      )     ->   List[Any]:
							A						   =   self.prepare_config_and_inputs()
							(
							    (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      (
							    A
							)   ,      
							)						   =   config_and_inputs
							A						   =   {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'lengths': input_lengths}
							return config, inputs_dict







@require_torch
class 				lowerCAmelCase_				(      _lowercase		,					_lowercase		,					_lowercase		,					unittest.TestCase    ):



						'''simple docstring'''


						_lowerCamelCase:    Dict      =    (
						    (
						        XLMModel,
						        XLMWithLMHeadModel,
						        XLMForQuestionAnswering,
						        XLMForSequenceClassification,
						        XLMForQuestionAnsweringSimple,
						        XLMForTokenClassification,
						        XLMForMultipleChoice,
						    )
						    if is_torch_available()
						    else ()
						)
						_lowerCamelCase:    Dict      =    (
						    (XLMWithLMHeadModel,) if is_torch_available() else ()
						)  # TODO (PVP): Check other models whether language generation is also applicable
						_lowerCamelCase:    Optional[Any]      =    (
						    {
						        '''feature-extraction''': XLMModel,
						        '''fill-mask''': XLMWithLMHeadModel,
						        '''question-answering''': XLMForQuestionAnsweringSimple,
						        '''text-classification''': XLMForSequenceClassification,
						        '''text-generation''': XLMWithLMHeadModel,
						        '''token-classification''': XLMForTokenClassification,
						        '''zero-shot''': XLMForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)


						def 						_SCREAMING_SNAKE_CASE      ( self     :      List[str]  ,A_     :      Optional[int]  ,A_     :      Optional[int]  ,A_     :      Tuple  ,A_     :      Any  ,A_     :      str      )     ->   Tuple:
							if (
							    pipeline_test_casse_name == "QAPipelineTests"
							    and tokenizer_name is not None
							    and not tokenizer_name.endswith('Fast'      )
							):
								# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
								# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
								# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
								return True

							return False


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Optional[int]  ,A_     :      Optional[int]  ,A_     :      Tuple  ,A_     :      str=False      )     ->   Dict:
							A						   =   super()._prepare_for_class(A_  ,A_  ,return_labels=A_      )

							if return_labels:
								if model_class.__name__ == "XLMForQuestionAnswering":
									A						   =   torch.zeros(
									    self.model_tester.batch_size  ,dtype=torch.long  ,device=A_      )
									A						   =   torch.zeros(
									    self.model_tester.batch_size  ,dtype=torch.long  ,device=A_      )

							return inputs_dict


						def 						_SCREAMING_SNAKE_CASE      ( self     :      int      )     ->   List[str]:
							A						   =   XLMModelTester(self      )
							A						   =   ConfigTester(self  ,config_class=A_  ,emb_dim=37      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Union[str, Any]      )     ->   str:
							self.config_tester.run_common_tests()


						def 						_SCREAMING_SNAKE_CASE      ( self     :      List[str]      )     ->   Optional[int]:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_model(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Any      )     ->   Dict:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_lm_head(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Union[str, Any]      )     ->   Dict:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_simple_qa(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Optional[Any]      )     ->   Tuple:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_qa(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      str      )     ->   List[Any]:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_sequence_classif(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      List[Any]      )     ->   Tuple:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_token_classif(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      int      )     ->   Dict:
							A						   =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_for_multiple_choice(*A_      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      Any  ,A_     :      Dict  ,A_     :      List[Any]  ,A_     :      List[Any]  ,A_     :      Dict  ,A_     :      Union[str, Any]  ,A_     :      List[Any]=False  ,A_     :      Tuple=1      )     ->   List[Any]:
							self.assertIsInstance(A_  ,A_      )
							self.assertListEqual(
							    [isinstance(A_  ,A_      ) for iter_attentions in attentions]  ,[True] * len(A_      )      )
							self.assertEqual(len(A_      )  ,(max_length - min_length) * num_beam_groups      )

							for idx, iter_attentions in enumerate(A_      ):
								# adds PAD dummy token
								A						   =   min_length + idx + 1
								A						   =   min_length + idx + 1

								A						   =   (
								    batch_size * num_beam_groups,
								    config.num_attention_heads,
								    tgt_len,
								    src_len,
								)
								# check attn size
								self.assertListEqual(
								    [layer_attention.shape for layer_attention in iter_attentions]  ,[expected_shape] * len(A_      )      )


						def 						_SCREAMING_SNAKE_CASE      ( self     :      List[str]  ,A_     :      Optional[Any]  ,A_     :      List[str]  ,A_     :      Dict  ,A_     :      Optional[Any]  ,A_     :      Dict  ,A_     :      Tuple=False  ,A_     :      Optional[Any]=1      )     ->   List[str]:
							self.assertIsInstance(A_  ,A_      )
							self.assertListEqual(
							    [isinstance(A_  ,A_      ) for iter_hidden_states in hidden_states]  ,[True] * len(A_      )  ,)
							self.assertEqual(len(A_      )  ,(max_length - min_length) * num_beam_groups      )

							for idx, iter_hidden_states in enumerate(A_      ):
								# adds PAD dummy token
								A						   =   min_length + idx + 1
								A						   =   (batch_size * num_beam_groups, seq_len, config.hidden_size)
								# check hidden size
								self.assertListEqual(
								    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]  ,[expected_shape] * len(A_      )  ,)
							pass




						@slow
						def 						_SCREAMING_SNAKE_CASE      ( self     :      Any      )     ->   List[str]:
							for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								A						   =   XLMModel.from_pretrained(A_      )
								self.assertIsNotNone(A_      )







@require_torch
class 				lowerCAmelCase_				(      unittest.TestCase    ):



						'''simple docstring'''


						@slow
						def 						_SCREAMING_SNAKE_CASE      ( self     :      Optional[Any]      )     ->   Dict:
							A						   =   XLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'      )
							model.to(A_      )
							A						   =   torch.tensor([[14, 447]]  ,dtype=torch.long  ,device=A_      )  # the president
							A						   =   [
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							    14,
							    447,
							]  # the president the president the president the president the president the president the president the president the president the president
							# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
							A						   =   model.generate(A_  ,do_sample=A_      )
							self.assertListEqual(output_ids[0].cpu().numpy().tolist()  ,A_      )