


'''simple docstring'''

import os
from pathlib import Path

def 							__magic_name__(						lowerCamelCase,				lowerCamelCase,				lowerCamelCase,				lowerCamelCase):
						__lowerCAmelCase							       =				{
						    '''en''': '''Machine learning is great, isn\'t it?''',
						    '''ru''': '''Машинное обучение - это здорово, не так ли?''',
						    '''de''': '''Maschinelles Lernen ist großartig, nicht wahr?''',
						}

						# BLUE scores as follows:
						# "pair": [fairseq, transformers]
						__lowerCAmelCase							       =				{
						    '''wmt16-en-de-dist-12-1''': [28.3, 27.52],
						    '''wmt16-en-de-dist-6-1''': [27.4, 27.11],
						    '''wmt16-en-de-12-1''': [26.9, 25.75],
						}
						__lowerCAmelCase							       =				F"""{src_lang}-{tgt_lang}"""

						__lowerCAmelCase							       =				F"""\n---\nlanguage:\n- {src_lang}\n- {tgt_lang}\nthumbnail:\ntags:\n- translation\n- wmt16\n- allenai\nlicense: apache-2.0\ndatasets:\n- wmt16\nmetrics:\n- bleu\n---\n\n# FSMT\n\n## Model description\n\nThis is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.\n\nFor more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).\n\nAll 3 models are available:\n\n* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)\n* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)\n* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)\n\n\n## Intended uses & limitations\n\n#### How to use\n\n```python\nfrom transformers import FSMTForConditionalGeneration, FSMTTokenizer\nmname = \"allenai/{model_name}\"\ntokenizer = FSMTTokenizer.from_pretrained(mname)\nmodel = FSMTForConditionalGeneration.from_pretrained(mname)\n\ninput = \"{texts[src_lang]}\"\ninput_ids = tokenizer.encode(input, return_tensors=\"pt\")\noutputs = model.generate(input_ids)\ndecoded = tokenizer.decode(outputs[0], skip_special_tokens=True)\nprint(decoded) # {texts[tgt_lang]}\n\n```\n\n#### Limitations and bias\n\n\n## Training data\n\nPretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).\n\n## Eval results\n\nHere are the BLEU scores:\n\nmodel   | fairseq | transformers\n-------|---------|----------\n{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}\n\nThe score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.\n\nThe score was calculated using this code:\n\n```bash\ngit clone https://github.com/huggingface/transformers\ncd transformers\nexport PAIR={pair}\nexport DATA_DIR=data/$PAIR\nexport SAVE_DIR=data/$PAIR\nexport BS=8\nexport NUM_BEAMS=5\nmkdir -p $DATA_DIR\nsacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source\nsacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target\necho $PAIR\nPYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS\n```\n\n## Data Sources\n\n- [training, etc.](http://www.statmt.org/wmt16/)\n- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)\n\n\n### BibTeX entry and citation info\n\n```\n@misc{{kasai2020deep,\n    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},\n    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},\n    year={{2020}},\n    eprint={{2006.10369}},\n    archivePrefix={{arXiv}},\n    primaryClass={{cs.CL}}\n}}\n```\n\n"""
						model_card_dir.mkdir(parents=a_,				exist_ok=a_)
						__lowerCAmelCase							       =				os.path.join(a_,				'''README.md''')
						print(F"""Generating {path}""")
						with open(a_,				'''w''',				encoding='''utf-8''') as f:
												f.write(a_)


# make sure we are under the root of the project
_UpperCAmelCase				: Tuple							    =				Path(__file__).resolve().parent.parent.parent
_UpperCAmelCase				: Union[str, Any]							    =				repo_dir / "model_cards"

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
	_UpperCAmelCase				: str							    =				model_cards_dir / "allenai" / model_name
	write_model_card(model_card_dir, src_lang="""en""", tgt_lang="""de""", model_name=model_name)





'''simple docstring'''

import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


_UpperCAmelCase				: Optional[Any]							    =				logging.getLogger(__name__)


class       a__				( __A      ):







				"""simple docstring"""



				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase       ,			__lowercase=None       ,			__lowercase=None		):
										__lowerCAmelCase							       =				self.layer[current_layer](__lowercase       ,			__lowercase       ,			head_mask[current_layer]		)

										__lowerCAmelCase							       =				layer_outputs[0]

										return hidden_states


@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'							,						__A							,						)
class       a__				( __A      ):







				"""simple docstring"""



				def __init__(self       ,			__lowercase		):
										super().__init__(__lowercase		)

										__lowerCAmelCase							       =				BertEncoderWithPabee(__lowercase		)

										self.init_weights()
										__lowerCAmelCase							       =				0
										__lowerCAmelCase							       =				0
										__lowerCAmelCase							       =				0

										__lowerCAmelCase							       =				0




				def 					_snake_case  (self       ,			__lowercase		):
										__lowerCAmelCase							       =				threshold




				def 					_snake_case  (self       ,			__lowercase		):
										__lowerCAmelCase							       =				patience




				def 					_snake_case  (self		):
										__lowerCAmelCase							       =				0
										__lowerCAmelCase							       =				0




				def 					_snake_case  (self		):
										__lowerCAmelCase							       =				self.inference_layers_num / self.inference_instances_num
										__lowerCAmelCase							       =				(
										    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
										    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
										)
										print(__lowercase		)




				@add_start_docstrings_to_model_forward(__lowercase		)
				def 					_snake_case  (self       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=False       ,			):

										if input_ids is not None and inputs_embeds is not None:
																raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''		)
										elif input_ids is not None:
																__lowerCAmelCase							       =				input_ids.size()
										elif inputs_embeds is not None:
																__lowerCAmelCase							       =				inputs_embeds.size()[:-1]
										else:
																raise ValueError('''You have to specify either input_ids or inputs_embeds'''		)

										__lowerCAmelCase							       =				input_ids.device if input_ids is not None else inputs_embeds.device

										if attention_mask is None:
																__lowerCAmelCase							       =				torch.ones(__lowercase       ,			device=__lowercase		)
										if token_type_ids is None:
																__lowerCAmelCase							       =				torch.zeros(__lowercase       ,			dtype=torch.long       ,			device=__lowercase		)

										# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
										# ourselves in which case we just need to make it broadcastable to all heads.
										__lowerCAmelCase							       =				self.get_extended_attention_mask(__lowercase       ,			__lowercase       ,			__lowercase		)

										# If a 2D ou 3D attention mask is provided for the cross-attention
										# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
										if self.config.is_decoder and encoder_hidden_states is not None:
																__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase							       =				encoder_hidden_states.size()
																__lowerCAmelCase							       =				(encoder_batch_size, encoder_sequence_length)
																if encoder_attention_mask is None:
																						__lowerCAmelCase							       =				torch.ones(__lowercase       ,			device=__lowercase		)
																__lowerCAmelCase							       =				self.invert_attention_mask(__lowercase		)
										else:
																__lowerCAmelCase							       =				None

										# Prepare head mask if needed
										# 1.0 in head_mask indicate we keep the head
										# attention_probs has shape bsz x n_heads x N x N
										# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
										# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
										__lowerCAmelCase							       =				self.get_head_mask(__lowercase       ,			self.config.num_hidden_layers		)

										__lowerCAmelCase							       =				self.embeddings(
										    input_ids=__lowercase       ,			position_ids=__lowercase       ,			token_type_ids=__lowercase       ,			inputs_embeds=__lowercase		)
										__lowerCAmelCase							       =				embedding_output

										if self.training:
																__lowerCAmelCase							       =				[]
																for i in range(self.config.num_hidden_layers		):
																						__lowerCAmelCase							       =				self.encoder.adaptive_forward(
																						    __lowercase       ,			current_layer=__lowercase       ,			attention_mask=__lowercase       ,			head_mask=__lowercase		)

																						__lowerCAmelCase							       =				self.pooler(__lowercase		)
																						__lowerCAmelCase							       =				output_layers[i](output_dropout(__lowercase		)		)
																						res.append(__lowercase		)
										elif self.patience == 0:  # Use all layers for inference
																__lowerCAmelCase							       =				self.encoder(
																    __lowercase       ,			attention_mask=__lowercase       ,			head_mask=__lowercase       ,			encoder_hidden_states=__lowercase       ,			encoder_attention_mask=__lowercase       ,			)
																__lowerCAmelCase							       =				self.pooler(encoder_outputs[0]		)
																__lowerCAmelCase							       =				[output_layers[self.config.num_hidden_layers - 1](__lowercase		)]
										else:
																__lowerCAmelCase							       =				0
																__lowerCAmelCase							       =				None
																__lowerCAmelCase							       =				0
																for i in range(self.config.num_hidden_layers		):
																						calculated_layer_num += 1
																						__lowerCAmelCase							       =				self.encoder.adaptive_forward(
																						    __lowercase       ,			current_layer=__lowercase       ,			attention_mask=__lowercase       ,			head_mask=__lowercase		)

																						__lowerCAmelCase							       =				self.pooler(__lowercase		)
																						__lowerCAmelCase							       =				output_layers[i](__lowercase		)
																						if regression:
																												__lowerCAmelCase							       =				logits.detach()
																												if patient_result is not None:
																																		__lowerCAmelCase							       =				patient_result.detach()
																												if (patient_result is not None) and torch.abs(patient_result - labels		) < self.regression_threshold:
																																		patient_counter += 1
																												else:
																																		__lowerCAmelCase							       =				0
																						else:
																												__lowerCAmelCase							       =				logits.detach().argmax(dim=1		)
																												if patient_result is not None:
																																		__lowerCAmelCase							       =				patient_result.detach().argmax(dim=1		)
																												if (patient_result is not None) and torch.all(labels.eq(__lowercase		)		):
																																		patient_counter += 1
																												else:
																																		__lowerCAmelCase							       =				0

																						__lowerCAmelCase							       =				logits
																						if patient_counter == self.patience:
																												break
																__lowerCAmelCase							       =				[patient_result]
																self.inference_layers_num += calculated_layer_num
																self.inference_instances_num += 1

										return res


@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '							,						__A							,						)
class       a__				( __A      ):







				"""simple docstring"""



				def __init__(self       ,			__lowercase		):
										super().__init__(__lowercase		)
										__lowerCAmelCase							       =				config.num_labels

										__lowerCAmelCase							       =				BertModelWithPabee(__lowercase		)
										__lowerCAmelCase							       =				nn.Dropout(config.hidden_dropout_prob		)
										__lowerCAmelCase							       =				nn.ModuleList(
										    [nn.Linear(config.hidden_size       ,			self.config.num_labels		) for _ in range(config.num_hidden_layers		)]		)

										self.init_weights()




				@add_start_docstrings_to_model_forward(__lowercase		)
				def 					_snake_case  (self       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			):
										__lowerCAmelCase							       =				self.bert(
										    input_ids=__lowercase       ,			attention_mask=__lowercase       ,			token_type_ids=__lowercase       ,			position_ids=__lowercase       ,			head_mask=__lowercase       ,			inputs_embeds=__lowercase       ,			output_dropout=self.dropout       ,			output_layers=self.classifiers       ,			regression=self.num_labels == 1       ,			)

										__lowerCAmelCase							       =				(logits[-1],)

										if labels is not None:
																__lowerCAmelCase							       =				None
																__lowerCAmelCase							       =				0
																for ix, logits_item in enumerate(__lowercase		):
																						if self.num_labels == 1:
																												#  We are doing regression
																												__lowerCAmelCase							       =				MSELoss()
																												__lowerCAmelCase							       =				loss_fct(logits_item.view(-1		)       ,			labels.view(-1		)		)
																						else:
																												__lowerCAmelCase							       =				CrossEntropyLoss()
																												__lowerCAmelCase							       =				loss_fct(logits_item.view(-1       ,			self.num_labels		)       ,			labels.view(-1		)		)
																						if total_loss is None:
																												__lowerCAmelCase							       =				loss
																						else:
																												total_loss += loss * (ix + 1)
																						total_weights += ix + 1
																__lowerCAmelCase							       =				(total_loss / total_weights,) + outputs

										return outputs

