




'''simple docstring'''

import warnings
from typing import Dict

import numpy as np

from ..utils import ExplicitEnum, add_end_docstrings, is_tf_available, is_torch_available
from .base import PIPELINE_INIT_ARGS, GenericTensor, Pipeline


if is_tf_available():
		from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

if is_torch_available():
		from ..models.auto.modeling_auto import MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING







def 						__UpperCamelCase				(							_lowercase					) ->			int:
	return 1.0 / (1.0 + np.exp(-_outputs					))







def 						__UpperCamelCase				(							_lowercase					) ->			List[str]:
	_lowercase					:       Union[str, Any]       =		np.max(_outputs,				axis=-1,				keepdims=_lowercase					)
	_lowercase					:       List[Any]       =		np.exp(_outputs - maxes					)
	return shifted_exp / shifted_exp.sum(axis=-1,				keepdims=_lowercase					)



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						"""sigmoid"""
					A_						    =						"""softmax"""
					A_						    =						"""none"""






@add_end_docstrings(
    A	,   R"""
        return_all_scores (`bool`, *optional*, defaults to `False`):
            Whether to return all prediction scores or just the one of the predicted class.
        function_to_apply (`str`, *optional*, defaults to `\"default\"`):
            The function to apply to the model outputs in order to retrieve the scores. Accepts four different values:

            - `\"default\"`: if the model has a single label, will apply the sigmoid function on the output. If the model
              has several labels, will apply the softmax function on the output.
            - `\"sigmoid\"`: Applies the sigmoid function on the output.
            - `\"softmax\"`: Applies the softmax function on the output.
            - `\"none\"`: Does not apply any function on the output.
    """	,   )
class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						False
					A_						    =						ClassificationFunction.NONE




					def __init__(  self				:	List[Any]     ,  **UpperCamelCase_				:	Tuple					)  ->       int:



						'''simple docstring'''





						super().__init__(**UpperCamelCase_					)

						self.check_model_type(
						    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
						    if self.framework == 'tf'
						    else MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING					)




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	str=None     ,  UpperCamelCase_				:	Union[str, Any]=None     ,  UpperCamelCase_				:	Any=""     ,  **UpperCamelCase_				:	Optional[Any]					)  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		tokenizer_kwargs

						_lowercase					:       Optional[Any]       =		{}
						if hasattr(self.model.config     ,  'return_all_scores'					) and return_all_scores is None:
							_lowercase					:       Union[str, Any]       =		self.model.config.return_all_scores

						if isinstance(UpperCamelCase_     ,  UpperCamelCase_					) or top_k is None:
							_lowercase					:       List[Any]       =		top_k
							_lowercase					:       Tuple       =		False
						elif return_all_scores is not None:
							warnings.warn(
							    '`return_all_scores` is now deprecated,  if want a similar functionality use `top_k=None` instead of'
							    ' `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.'     ,  UpperCamelCase_     ,  )
							if return_all_scores:
								_lowercase					:       Optional[int]       =		None
							else:
								_lowercase					:       int       =		1

						if isinstance(UpperCamelCase_     ,  UpperCamelCase_					):
							_lowercase					:       Dict       =		ClassificationFunction[function_to_apply.upper()]

						if function_to_apply is not None:
							_lowercase					:       Optional[Any]       =		function_to_apply
						return preprocess_params, {}, postprocess_params




					def __call__(  self				:	Tuple     ,  *UpperCamelCase_				:	List[str]     ,  **UpperCamelCase_				:	Union[str, Any]					)  ->       str:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		super().__call__(*UpperCamelCase_     ,  **UpperCamelCase_					)
						# TODO try and retrieve it in a nicer way from _sanitize_parameters.
						_lowercase					:       Union[str, Any]       =		'top_k' not in kwargs
						if isinstance(args[0]     ,  UpperCamelCase_					) and _legacy:
							# This pipeline is odd, and return a list when single item is run
							return [result]
						else:
							return result




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	Union[str, Any]     ,  **UpperCamelCase_				:	Union[str, Any]					)  ->       Dict[str, GenericTensor]:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		self.framework
						if isinstance(UpperCamelCase_     ,  UpperCamelCase_					):
							return self.tokenizer(**UpperCamelCase_     ,  return_tensors=UpperCamelCase_     ,  **UpperCamelCase_					)
						elif isinstance(UpperCamelCase_     ,  UpperCamelCase_					) and len(UpperCamelCase_					) == 1 and isinstance(inputs[0]     ,  UpperCamelCase_					) and len(inputs[0]					) == 2:
							# It used to be valid to use a list of list of list for text pairs, keeping this path for BC
							return self.tokenizer(
							    text=inputs[0][0]     ,  text_pair=inputs[0][1]     ,  return_tensors=UpperCamelCase_     ,  **UpperCamelCase_					)
						elif isinstance(UpperCamelCase_     ,  UpperCamelCase_					):
							# This is likely an invalid usage of the pipeline attempting to pass text pairs.
							raise ValueError(
							    'The pipeline received invalid inputs, if you are trying to send text pairs, you can try to send a'
							    ' dictionary `{"text": "My text", "text_pair": "My pair"}` in order to send a text pair.'					)
						return self.tokenizer(UpperCamelCase_     ,  return_tensors=UpperCamelCase_     ,  **UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	List[Any]					)  ->       Tuple:



						'''simple docstring'''





						return self.model(**UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	Optional[int]     ,  UpperCamelCase_				:	List[Any]=None     ,  UpperCamelCase_				:	Optional[int]=1     ,  UpperCamelCase_				:	List[Any]=True					)  ->       Union[str, Any]:



						'''simple docstring'''





						if function_to_apply is None:
							if self.model.config.problem_type == "multi_label_classification" or self.model.config.num_labels == 1:
								_lowercase					:       Any       =		ClassificationFunction.SIGMOID
							elif self.model.config.problem_type == "single_label_classification" or self.model.config.num_labels > 1:
								_lowercase					:       str       =		ClassificationFunction.SOFTMAX
							elif hasattr(self.model.config     ,  'function_to_apply'					) and function_to_apply is None:
								_lowercase					:       List[Any]       =		self.model.config.function_to_apply
							else:
								_lowercase					:       List[str]       =		ClassificationFunction.NONE

						_lowercase					:       Optional[Any]       =		model_outputs['logits'][0]
						_lowercase					:       int       =		outputs.numpy()

						if function_to_apply == ClassificationFunction.SIGMOID:
							_lowercase					:       Union[str, Any]       =		sigmoid(UpperCamelCase_					)
						elif function_to_apply == ClassificationFunction.SOFTMAX:
							_lowercase					:       List[Any]       =		softmax(UpperCamelCase_					)
						elif function_to_apply == ClassificationFunction.NONE:
							_lowercase					:       List[str]       =		outputs
						else:
							raise ValueError(F'''Unrecognized `function_to_apply` argument: {function_to_apply}'''					)

						if top_k == 1 and _legacy:
							return {"label": self.model.config.idalabel[scores.argmax().item()], "score": scores.max().item()}

						_lowercase					:       Union[str, Any]       =		[
						    {'label': self.model.config.idalabel[i], 'score': score.item()} for i, score in enumerate(UpperCamelCase_					)
						]
						if not _legacy:
							dict_scores.sort(key=lambda UpperCamelCase_					: x["score"]     ,  reverse=UpperCamelCase_					)
							if top_k is not None:
								_lowercase					:       Any       =		dict_scores[:top_k]
						return dict_scores





'''simple docstring'''

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)







def 						__UpperCamelCase				(							_lowercase					) ->			Tuple:
	_lowercase					:       int       =		torch.exp(_lowercase					)
	_lowercase					:       List[str]       =		torch.sum(_lowercase,				dim=1					)  # sum of exp(x_i)
	_lowercase					:       str       =		torch.sum(x * exp_x,				dim=1					)  # sum of x_i * exp(x_i)
	return torch.log(_lowercase					) - B / A



class 		lowerCamelCase__      (						nn.Module     ):




					'''simple docstring'''



					def __init__(  self				:	Optional[Any]     ,  UpperCamelCase_				:	List[str]					)  ->       Optional[Any]:



						'''simple docstring'''





						super().__init__()
						_lowercase					:       int       =		config.output_attentions
						_lowercase					:       int       =		config.output_hidden_states
						_lowercase					:       Union[str, Any]       =		nn.ModuleList([BertLayer(UpperCamelCase_					) for _ in range(config.num_hidden_layers					)]					)
						_lowercase					:       List[Any]       =		nn.ModuleList([BertHighway(UpperCamelCase_					) for _ in range(config.num_hidden_layers					)]					)

						_lowercase					:       Tuple       =		[-1 for _ in range(config.num_hidden_layers					)]




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	str					)  ->       int:



						'''simple docstring'''





						if (type(UpperCamelCase_					) is float) or (type(UpperCamelCase_					) is int):
							for i in range(len(self.early_exit_entropy					)					):
								_lowercase					:       Optional[Any]       =		x
						else:
							_lowercase					:       Optional[int]       =		x




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	List[Any]					)  ->       Dict:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		pooler.state_dict()
						for highway in self.highway:
							for name, param in highway.pooler.state_dict().items():
								param.copy_(loaded_model[name]					)




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	List[Any]=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Union[str, Any]=None     ,  UpperCamelCase_				:	Optional[Any]=None     ,  )  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       int       =		()
						_lowercase					:       List[Any]       =		()
						_lowercase					:       Tuple       =		()
						for i, layer_module in enumerate(self.layer					):
							if self.output_hidden_states:
								_lowercase					:       Optional[int]       =		all_hidden_states + (hidden_states,)

							_lowercase					:       str       =		layer_module(
							    UpperCamelCase_     ,  UpperCamelCase_     ,  head_mask[i]     ,  UpperCamelCase_     ,  UpperCamelCase_					)
							_lowercase					:       List[str]       =		layer_outputs[0]

							if self.output_attentions:
								_lowercase					:       Tuple       =		all_attentions + (layer_outputs[1],)

							_lowercase					:       Optional[int]       =		(hidden_states,)
							if self.output_hidden_states:
								_lowercase					:       str       =		current_outputs + (all_hidden_states,)
							if self.output_attentions:
								_lowercase					:       Optional[int]       =		current_outputs + (all_attentions,)

							_lowercase					:       List[Any]       =		self.highway[i](UpperCamelCase_					)
							# logits, pooled_output

							if not self.training:
								_lowercase					:       Dict       =		highway_exit[0]
								_lowercase					:       Tuple       =		entropy(UpperCamelCase_					)
								_lowercase					:       Dict       =		highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
								_lowercase					:       str       =		all_highway_exits + (highway_exit,)

								if highway_entropy < self.early_exit_entropy[i]:
									_lowercase					:       Tuple       =		(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
									raise HighwayException(UpperCamelCase_     ,  i + 1					)
							else:
								_lowercase					:       Optional[int]       =		all_highway_exits + (highway_exit,)

        # Add last layer
						if self.output_hidden_states:
							_lowercase					:       str       =		all_hidden_states + (hidden_states,)

						_lowercase					:       Optional[Any]       =		(hidden_states,)
						if self.output_hidden_states:
							_lowercase					:       Dict       =		outputs + (all_hidden_states,)
						if self.output_attentions:
							_lowercase					:       Optional[Any]       =		outputs + (all_attentions,)

						_lowercase					:       Optional[int]       =		outputs + (all_highway_exits,)
						return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits






@add_start_docstrings(
    """The Bert Model transformer with early exiting (DeeBERT). """	,   A	,   )
class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	Optional[int]					)  ->       Optional[Any]:



						'''simple docstring'''





						super().__init__(UpperCamelCase_					)
						_lowercase					:       int       =		config

						_lowercase					:       int       =		BertEmbeddings(UpperCamelCase_					)
						_lowercase					:       List[Any]       =		DeeBertEncoder(UpperCamelCase_					)
						_lowercase					:       Any       =		BertPooler(UpperCamelCase_					)

						self.init_weights()




					def 						__UpperCAmelCase     (  self				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						self.encoder.init_highway_pooler(self.pooler					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       List[str]:



						'''simple docstring'''





						return self.embeddings.word_embeddings




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Dict					)  ->       Any:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		value




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						for layer, heads in heads_to_prune.items():
							self.encoder.layer[layer].attention.prune_heads(UpperCamelCase_					)




					@add_start_docstrings_to_model_forward(UpperCamelCase_					)
					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	Optional[Any]=None     ,  UpperCamelCase_				:	str=None     ,  UpperCamelCase_				:	Optional[Any]=None     ,  UpperCamelCase_				:	Tuple=None     ,  UpperCamelCase_				:	Any=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	int=None     ,  UpperCamelCase_				:	Tuple=None     ,  )  ->       Union[str, Any]:



						'''simple docstring'''





						if input_ids is not None and inputs_embeds is not None:
							raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'					)
						elif input_ids is not None:
							_lowercase					:       Any       =		input_ids.size()
						elif inputs_embeds is not None:
							_lowercase					:       Any       =		inputs_embeds.size()[:-1]
						else:
							raise ValueError('You have to specify either input_ids or inputs_embeds'					)

						_lowercase					:       str       =		input_ids.device if input_ids is not None else inputs_embeds.device

						if attention_mask is None:
							_lowercase					:       Tuple       =		torch.ones(UpperCamelCase_     ,  device=UpperCamelCase_					)
						if encoder_attention_mask is None:
							_lowercase					:       Dict       =		torch.ones(UpperCamelCase_     ,  device=UpperCamelCase_					)
						if token_type_ids is None:
							_lowercase					:       int       =		torch.zeros(UpperCamelCase_     ,  dtype=torch.long     ,  device=UpperCamelCase_					)

						# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
						# ourselves in which case we just need to make it broadcastable to all heads.
						_lowercase					:       torch.Tensor       =		self.get_extended_attention_mask(UpperCamelCase_     ,  UpperCamelCase_     ,  UpperCamelCase_					)

						# If a 2D ou 3D attention mask is provided for the cross-attention
						# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
						if encoder_attention_mask.dim() == 3:
							_lowercase					:       int       =		encoder_attention_mask[:, None, :, :]
						if encoder_attention_mask.dim() == 2:
							_lowercase					:       int       =		encoder_attention_mask[:, None, None, :]

						_lowercase					:       str       =		encoder_extended_attention_mask.to(
						    dtype=next(self.parameters()					).dtype					)  # fp16 compatibility
						_lowercase					:       Optional[int]       =		(1.0 - encoder_extended_attention_mask) * -1_00_00.0

						# Prepare head mask if needed
						# 1.0 in head_mask indicate we keep the head
						# attention_probs has shape bsz x n_heads x N x N
						# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
						# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
						_lowercase					:       Optional[int]       =		self.get_head_mask(UpperCamelCase_     ,  self.config.num_hidden_layers					)

						_lowercase					:       Dict       =		self.embeddings(
						    input_ids=UpperCamelCase_     ,  position_ids=UpperCamelCase_     ,  token_type_ids=UpperCamelCase_     ,  inputs_embeds=UpperCamelCase_					)
						_lowercase					:       List[Any]       =		self.encoder(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  head_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  encoder_attention_mask=UpperCamelCase_     ,  )
						_lowercase					:       int       =		encoder_outputs[0]
						_lowercase					:       str       =		self.pooler(UpperCamelCase_					)

						_lowercase					:       List[Any]       =		(
						    sequence_output,
						    pooled_output,
						) + encoder_outputs[
						    1:
						]  # add hidden_states and attentions if they are here
						return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits






class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	Dict     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	Dict					)  ->       Optional[Any]:



						'''simple docstring'''





						_lowercase					:       Any       =		message
						_lowercase					:       Dict       =		exit_layer  # start from 1!






class 		lowerCamelCase__      (						nn.Module     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	List[str]					)  ->       Dict:



						'''simple docstring'''





						super().__init__()
						_lowercase					:       Optional[Any]       =		BertPooler(UpperCamelCase_					)
						_lowercase					:       List[Any]       =		nn.Dropout(config.hidden_dropout_prob					)
						_lowercase					:       int       =		nn.Linear(config.hidden_size     ,  config.num_labels					)




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	Optional[int]					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       str       =		encoder_outputs[0]
						_lowercase					:       int       =		self.pooler(UpperCamelCase_					)
						# "return" pooler_output

						# BertModel
						_lowercase					:       Optional[int]       =		(pooler_input, pooler_output) + encoder_outputs[1:]
						# "return" bmodel_output

						# Dropout and classification
						_lowercase					:       Dict       =		bmodel_output[1]

						_lowercase					:       Union[str, Any]       =		self.dropout(UpperCamelCase_					)
						_lowercase					:       str       =		self.classifier(UpperCamelCase_					)

						return logits, pooled_output






@add_start_docstrings(
    """Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. """	,   A	,   )
class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	List[Any]					)  ->       List[str]:



						'''simple docstring'''





						super().__init__(UpperCamelCase_					)
						_lowercase					:       Dict       =		config.num_labels
						_lowercase					:       Any       =		config.num_hidden_layers

						_lowercase					:       Optional[int]       =		DeeBertModel(UpperCamelCase_					)
						_lowercase					:       Any       =		nn.Dropout(config.hidden_dropout_prob					)
						_lowercase					:       Optional[Any]       =		nn.Linear(config.hidden_size     ,  self.config.num_labels					)

						self.init_weights()




					@add_start_docstrings_to_model_forward(UpperCamelCase_					)
					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Dict=None     ,  UpperCamelCase_				:	Union[str, Any]=None     ,  UpperCamelCase_				:	List[Any]=None     ,  UpperCamelCase_				:	Tuple=None     ,  UpperCamelCase_				:	Any=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	str=-1     ,  UpperCamelCase_				:	Union[str, Any]=False     ,  )  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       Union[str, Any]       =		self.num_layers
						try:
							_lowercase					:       Tuple       =		self.bert(
							    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  token_type_ids=UpperCamelCase_     ,  position_ids=UpperCamelCase_     ,  head_mask=UpperCamelCase_     ,  inputs_embeds=UpperCamelCase_     ,  )
							# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

							_lowercase					:       List[Any]       =		outputs[1]

							_lowercase					:       int       =		self.dropout(UpperCamelCase_					)
							_lowercase					:       Optional[int]       =		self.classifier(UpperCamelCase_					)
							_lowercase					:       Union[str, Any]       =		(logits,) + outputs[2:]  # add hidden states and attention if they are here
						except HighwayException as e:
							_lowercase					:       Union[str, Any]       =		e.message
							_lowercase					:       Any       =		e.exit_layer
							_lowercase					:       Optional[int]       =		outputs[0]

						if not self.training:
							_lowercase					:       Union[str, Any]       =		entropy(UpperCamelCase_					)
							_lowercase					:       Tuple       =		[]
							_lowercase					:       Tuple       =		[]
						if labels is not None:
							if self.num_labels == 1:
								#  We are doing regression
								_lowercase					:       Tuple       =		MSELoss()
								_lowercase					:       Tuple       =		loss_fct(logits.view(-1					)     ,  labels.view(-1					)					)
							else:
								_lowercase					:       Union[str, Any]       =		CrossEntropyLoss()
								_lowercase					:       Tuple       =		loss_fct(logits.view(-1     ,  self.num_labels					)     ,  labels.view(-1					)					)

							# work with highway exits
							_lowercase					:       Optional[Any]       =		[]
							for highway_exit in outputs[-1]:
								_lowercase					:       Optional[Any]       =		highway_exit[0]
								if not self.training:
									highway_logits_all.append(UpperCamelCase_					)
									highway_entropy.append(highway_exit[2]					)
								if self.num_labels == 1:
									#  We are doing regression
									_lowercase					:       Union[str, Any]       =		MSELoss()
									_lowercase					:       Any       =		loss_fct(highway_logits.view(-1					)     ,  labels.view(-1					)					)
								else:
									_lowercase					:       Dict       =		CrossEntropyLoss()
									_lowercase					:       Optional[int]       =		loss_fct(highway_logits.view(-1     ,  self.num_labels					)     ,  labels.view(-1					)					)
								highway_losses.append(UpperCamelCase_					)

							if train_highway:
								_lowercase					:       List[str]       =		(sum(highway_losses[:-1]					),) + outputs
								# exclude the final highway, of course
							else:
								_lowercase					:       Optional[Any]       =		(loss,) + outputs
						if not self.training:
							_lowercase					:       List[Any]       =		outputs + ((original_entropy, highway_entropy), exit_layer)
							if output_layer >= 0:
								_lowercase					:       Dict       =		(
								    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
								)  # use the highway of the last layer

						return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)
