



'''simple docstring'''

from scipy.stats import pearsonr

import datasets


_A				:    Optional[Any]     						='''
Pearson correlation coefficient and p-value for testing non-correlation.
The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases.
The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets.
'''


_A				:    Dict     						='''
Args:
    predictions (`list` of `int`): Predicted class labels, as returned by a model.
    references (`list` of `int`): Ground truth labels.
    return_pvalue (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`.

Returns:
    pearsonr (`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation.
    p-value (`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities.

Examples:

    Example 1-A simple example using only predictions and references.
        >>> pearsonr_metric = datasets.load_metric("pearsonr")
        >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5])
        >>> print(round(results[\'pearsonr\'], 2))
        -0.74

    Example 2-The same as Example 1, but that also returns the `p-value`.
        >>> pearsonr_metric = datasets.load_metric("pearsonr")
        >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True)
        >>> print(sorted(list(results.keys())))
        [\'p-value\', \'pearsonr\']
        >>> print(round(results[\'pearsonr\'], 2))
        -0.74
        >>> print(round(results[\'p-value\'], 2))
        0.15
'''


_A				:    List[Any]     						='''
@article{2020SciPy-NMeth,
author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and
      Haberland, Matt and Reddy, Tyler and Cournapeau, David and
      Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and
      Bright, Jonathan and {van der Walt}, St{\'e}fan J. and
      Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and
      Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and
      Kern, Robert and Larson, Eric and Carey, C J and
      Polat, Ilhan and Feng, Yu and Moore, Eric W. and
      {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and
      Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and
      Harris, Charles R. and Archibald, Anne M. and
      Ribeiro, Antonio H. and Pedregosa, Fabian and
      {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},
title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific
      Computing in Python}},
journal = {Nature Methods},
year    = {2020},
volume  = {17},
pages   = {261--272},
adsurl  = {https://rdcu.be/b08Wh},
doi = {10.1038/s41592-019-0686-2},
}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,   _KWARGS_DESCRIPTION     )
class 		lowerCamelCase__      (						datasets.Metric     ):




					'''simple docstring'''



					def 						__UpperCAmelCase     (  self				:	str					)  ->       List[str]:



						'''simple docstring'''





						return datasets.MetricInfo(
						    description=_DESCRIPTION     ,  citation=_CITATION     ,  inputs_description=_KWARGS_DESCRIPTION     ,  features=datasets.Features(
						        {
						            'predictions': datasets.Value('float'					),
						            'references': datasets.Value('float'					),
						        }					)     ,  reference_urls=['https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html']     ,  )




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	int     ,  UpperCamelCase_				:	List[str]=False					)  ->       List[str]:



						'''simple docstring'''





						if return_pvalue:
							_lowercase					:       Tuple       =		pearsonr(UpperCamelCase_     ,  UpperCamelCase_					)
							return {"pearsonr": results[0], "p-value": results[1]}
						else:
							return {"pearsonr": float(pearsonr(UpperCamelCase_     ,  UpperCamelCase_					)[0]					)}





'''simple docstring'''

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)







def 						__UpperCamelCase				(							_lowercase					) ->			Tuple:
	_lowercase					:       int       =		torch.exp(_lowercase					)
	_lowercase					:       List[str]       =		torch.sum(_lowercase,				dim=1					)  # sum of exp(x_i)
	_lowercase					:       str       =		torch.sum(x * exp_x,				dim=1					)  # sum of x_i * exp(x_i)
	return torch.log(_lowercase					) - B / A



class 		lowerCamelCase__      (						nn.Module     ):




					'''simple docstring'''



					def __init__(  self				:	Optional[Any]     ,  UpperCamelCase_				:	List[str]					)  ->       Optional[Any]:



						'''simple docstring'''





						super().__init__()
						_lowercase					:       int       =		config.output_attentions
						_lowercase					:       int       =		config.output_hidden_states
						_lowercase					:       Union[str, Any]       =		nn.ModuleList([BertLayer(UpperCamelCase_					) for _ in range(config.num_hidden_layers					)]					)
						_lowercase					:       List[Any]       =		nn.ModuleList([BertHighway(UpperCamelCase_					) for _ in range(config.num_hidden_layers					)]					)

						_lowercase					:       Tuple       =		[-1 for _ in range(config.num_hidden_layers					)]




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	str					)  ->       int:



						'''simple docstring'''





						if (type(UpperCamelCase_					) is float) or (type(UpperCamelCase_					) is int):
							for i in range(len(self.early_exit_entropy					)					):
								_lowercase					:       Optional[Any]       =		x
						else:
							_lowercase					:       Optional[int]       =		x




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	List[Any]					)  ->       Dict:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		pooler.state_dict()
						for highway in self.highway:
							for name, param in highway.pooler.state_dict().items():
								param.copy_(loaded_model[name]					)




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	List[Any]=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Union[str, Any]=None     ,  UpperCamelCase_				:	Optional[Any]=None     ,  )  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       int       =		()
						_lowercase					:       List[Any]       =		()
						_lowercase					:       Tuple       =		()
						for i, layer_module in enumerate(self.layer					):
							if self.output_hidden_states:
								_lowercase					:       Optional[int]       =		all_hidden_states + (hidden_states,)

							_lowercase					:       str       =		layer_module(
							    UpperCamelCase_     ,  UpperCamelCase_     ,  head_mask[i]     ,  UpperCamelCase_     ,  UpperCamelCase_					)
							_lowercase					:       List[str]       =		layer_outputs[0]

							if self.output_attentions:
								_lowercase					:       Tuple       =		all_attentions + (layer_outputs[1],)

							_lowercase					:       Optional[int]       =		(hidden_states,)
							if self.output_hidden_states:
								_lowercase					:       str       =		current_outputs + (all_hidden_states,)
							if self.output_attentions:
								_lowercase					:       Optional[int]       =		current_outputs + (all_attentions,)

							_lowercase					:       List[Any]       =		self.highway[i](UpperCamelCase_					)
							# logits, pooled_output

							if not self.training:
								_lowercase					:       Dict       =		highway_exit[0]
								_lowercase					:       Tuple       =		entropy(UpperCamelCase_					)
								_lowercase					:       Dict       =		highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
								_lowercase					:       str       =		all_highway_exits + (highway_exit,)

								if highway_entropy < self.early_exit_entropy[i]:
									_lowercase					:       Tuple       =		(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
									raise HighwayException(UpperCamelCase_     ,  i + 1					)
							else:
								_lowercase					:       Optional[int]       =		all_highway_exits + (highway_exit,)

        # Add last layer
						if self.output_hidden_states:
							_lowercase					:       str       =		all_hidden_states + (hidden_states,)

						_lowercase					:       Optional[Any]       =		(hidden_states,)
						if self.output_hidden_states:
							_lowercase					:       Dict       =		outputs + (all_hidden_states,)
						if self.output_attentions:
							_lowercase					:       Optional[Any]       =		outputs + (all_attentions,)

						_lowercase					:       Optional[int]       =		outputs + (all_highway_exits,)
						return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits






@add_start_docstrings(
    """The Bert Model transformer with early exiting (DeeBERT). """	,   A	,   )
class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	Optional[int]					)  ->       Optional[Any]:



						'''simple docstring'''





						super().__init__(UpperCamelCase_					)
						_lowercase					:       int       =		config

						_lowercase					:       int       =		BertEmbeddings(UpperCamelCase_					)
						_lowercase					:       List[Any]       =		DeeBertEncoder(UpperCamelCase_					)
						_lowercase					:       Any       =		BertPooler(UpperCamelCase_					)

						self.init_weights()




					def 						__UpperCAmelCase     (  self				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						self.encoder.init_highway_pooler(self.pooler					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       List[str]:



						'''simple docstring'''





						return self.embeddings.word_embeddings




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Dict					)  ->       Any:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		value




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						for layer, heads in heads_to_prune.items():
							self.encoder.layer[layer].attention.prune_heads(UpperCamelCase_					)




					@add_start_docstrings_to_model_forward(UpperCamelCase_					)
					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	Optional[Any]=None     ,  UpperCamelCase_				:	str=None     ,  UpperCamelCase_				:	Optional[Any]=None     ,  UpperCamelCase_				:	Tuple=None     ,  UpperCamelCase_				:	Any=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	int=None     ,  UpperCamelCase_				:	Tuple=None     ,  )  ->       Union[str, Any]:



						'''simple docstring'''





						if input_ids is not None and inputs_embeds is not None:
							raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'					)
						elif input_ids is not None:
							_lowercase					:       Any       =		input_ids.size()
						elif inputs_embeds is not None:
							_lowercase					:       Any       =		inputs_embeds.size()[:-1]
						else:
							raise ValueError('You have to specify either input_ids or inputs_embeds'					)

						_lowercase					:       str       =		input_ids.device if input_ids is not None else inputs_embeds.device

						if attention_mask is None:
							_lowercase					:       Tuple       =		torch.ones(UpperCamelCase_     ,  device=UpperCamelCase_					)
						if encoder_attention_mask is None:
							_lowercase					:       Dict       =		torch.ones(UpperCamelCase_     ,  device=UpperCamelCase_					)
						if token_type_ids is None:
							_lowercase					:       int       =		torch.zeros(UpperCamelCase_     ,  dtype=torch.long     ,  device=UpperCamelCase_					)

						# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
						# ourselves in which case we just need to make it broadcastable to all heads.
						_lowercase					:       torch.Tensor       =		self.get_extended_attention_mask(UpperCamelCase_     ,  UpperCamelCase_     ,  UpperCamelCase_					)

						# If a 2D ou 3D attention mask is provided for the cross-attention
						# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
						if encoder_attention_mask.dim() == 3:
							_lowercase					:       int       =		encoder_attention_mask[:, None, :, :]
						if encoder_attention_mask.dim() == 2:
							_lowercase					:       int       =		encoder_attention_mask[:, None, None, :]

						_lowercase					:       str       =		encoder_extended_attention_mask.to(
						    dtype=next(self.parameters()					).dtype					)  # fp16 compatibility
						_lowercase					:       Optional[int]       =		(1.0 - encoder_extended_attention_mask) * -1_00_00.0

						# Prepare head mask if needed
						# 1.0 in head_mask indicate we keep the head
						# attention_probs has shape bsz x n_heads x N x N
						# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
						# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
						_lowercase					:       Optional[int]       =		self.get_head_mask(UpperCamelCase_     ,  self.config.num_hidden_layers					)

						_lowercase					:       Dict       =		self.embeddings(
						    input_ids=UpperCamelCase_     ,  position_ids=UpperCamelCase_     ,  token_type_ids=UpperCamelCase_     ,  inputs_embeds=UpperCamelCase_					)
						_lowercase					:       List[Any]       =		self.encoder(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  head_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  encoder_attention_mask=UpperCamelCase_     ,  )
						_lowercase					:       int       =		encoder_outputs[0]
						_lowercase					:       str       =		self.pooler(UpperCamelCase_					)

						_lowercase					:       List[Any]       =		(
						    sequence_output,
						    pooled_output,
						) + encoder_outputs[
						    1:
						]  # add hidden_states and attentions if they are here
						return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits






class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	Dict     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	Dict					)  ->       Optional[Any]:



						'''simple docstring'''





						_lowercase					:       Any       =		message
						_lowercase					:       Dict       =		exit_layer  # start from 1!






class 		lowerCamelCase__      (						nn.Module     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	List[str]					)  ->       Dict:



						'''simple docstring'''





						super().__init__()
						_lowercase					:       Optional[Any]       =		BertPooler(UpperCamelCase_					)
						_lowercase					:       List[Any]       =		nn.Dropout(config.hidden_dropout_prob					)
						_lowercase					:       int       =		nn.Linear(config.hidden_size     ,  config.num_labels					)




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	Optional[int]					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       str       =		encoder_outputs[0]
						_lowercase					:       int       =		self.pooler(UpperCamelCase_					)
						# "return" pooler_output

						# BertModel
						_lowercase					:       Optional[int]       =		(pooler_input, pooler_output) + encoder_outputs[1:]
						# "return" bmodel_output

						# Dropout and classification
						_lowercase					:       Dict       =		bmodel_output[1]

						_lowercase					:       Union[str, Any]       =		self.dropout(UpperCamelCase_					)
						_lowercase					:       str       =		self.classifier(UpperCamelCase_					)

						return logits, pooled_output






@add_start_docstrings(
    """Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. """	,   A	,   )
class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					def __init__(  self				:	int     ,  UpperCamelCase_				:	List[Any]					)  ->       List[str]:



						'''simple docstring'''





						super().__init__(UpperCamelCase_					)
						_lowercase					:       Dict       =		config.num_labels
						_lowercase					:       Any       =		config.num_hidden_layers

						_lowercase					:       Optional[int]       =		DeeBertModel(UpperCamelCase_					)
						_lowercase					:       Any       =		nn.Dropout(config.hidden_dropout_prob					)
						_lowercase					:       Optional[Any]       =		nn.Linear(config.hidden_size     ,  self.config.num_labels					)

						self.init_weights()




					@add_start_docstrings_to_model_forward(UpperCamelCase_					)
					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Dict=None     ,  UpperCamelCase_				:	Union[str, Any]=None     ,  UpperCamelCase_				:	List[Any]=None     ,  UpperCamelCase_				:	Tuple=None     ,  UpperCamelCase_				:	Any=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	str=-1     ,  UpperCamelCase_				:	Union[str, Any]=False     ,  )  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       Union[str, Any]       =		self.num_layers
						try:
							_lowercase					:       Tuple       =		self.bert(
							    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  token_type_ids=UpperCamelCase_     ,  position_ids=UpperCamelCase_     ,  head_mask=UpperCamelCase_     ,  inputs_embeds=UpperCamelCase_     ,  )
							# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

							_lowercase					:       List[Any]       =		outputs[1]

							_lowercase					:       int       =		self.dropout(UpperCamelCase_					)
							_lowercase					:       Optional[int]       =		self.classifier(UpperCamelCase_					)
							_lowercase					:       Union[str, Any]       =		(logits,) + outputs[2:]  # add hidden states and attention if they are here
						except HighwayException as e:
							_lowercase					:       Union[str, Any]       =		e.message
							_lowercase					:       Any       =		e.exit_layer
							_lowercase					:       Optional[int]       =		outputs[0]

						if not self.training:
							_lowercase					:       Union[str, Any]       =		entropy(UpperCamelCase_					)
							_lowercase					:       Tuple       =		[]
							_lowercase					:       Tuple       =		[]
						if labels is not None:
							if self.num_labels == 1:
								#  We are doing regression
								_lowercase					:       Tuple       =		MSELoss()
								_lowercase					:       Tuple       =		loss_fct(logits.view(-1					)     ,  labels.view(-1					)					)
							else:
								_lowercase					:       Union[str, Any]       =		CrossEntropyLoss()
								_lowercase					:       Tuple       =		loss_fct(logits.view(-1     ,  self.num_labels					)     ,  labels.view(-1					)					)

							# work with highway exits
							_lowercase					:       Optional[Any]       =		[]
							for highway_exit in outputs[-1]:
								_lowercase					:       Optional[Any]       =		highway_exit[0]
								if not self.training:
									highway_logits_all.append(UpperCamelCase_					)
									highway_entropy.append(highway_exit[2]					)
								if self.num_labels == 1:
									#  We are doing regression
									_lowercase					:       Union[str, Any]       =		MSELoss()
									_lowercase					:       Any       =		loss_fct(highway_logits.view(-1					)     ,  labels.view(-1					)					)
								else:
									_lowercase					:       Dict       =		CrossEntropyLoss()
									_lowercase					:       Optional[int]       =		loss_fct(highway_logits.view(-1     ,  self.num_labels					)     ,  labels.view(-1					)					)
								highway_losses.append(UpperCamelCase_					)

							if train_highway:
								_lowercase					:       List[str]       =		(sum(highway_losses[:-1]					),) + outputs
								# exclude the final highway, of course
							else:
								_lowercase					:       Optional[Any]       =		(loss,) + outputs
						if not self.training:
							_lowercase					:       List[Any]       =		outputs + ((original_entropy, highway_entropy), exit_layer)
							if output_layer >= 0:
								_lowercase					:       Dict       =		(
								    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
								)  # use the highway of the last layer

						return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)
