




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy


@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """     ,					__UpperCAmelCase     ,					)
class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: List[str]										=RobertaConfig
				__a							: Optional[Any]										="""roberta"""



				def __init__( self       , UpperCAmelCase_						):
											super().__init__(UpperCAmelCase_						)

											lowerCAmelCase   =		RobertaEmbeddings(UpperCAmelCase_						)
											self.init_weights()




@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """     ,					__UpperCAmelCase     ,					)
class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: List[str]										=RobertaConfig
				__a							: List[Any]										="""roberta"""



				def __init__( self       , UpperCAmelCase_						):
											super().__init__(UpperCAmelCase_						)
											lowerCAmelCase   =		config.num_labels
											lowerCAmelCase   =		config.num_hidden_layers

											lowerCAmelCase   =		DeeRobertaModel(UpperCAmelCase_						)
											lowerCAmelCase   =		nn.Dropout(config.hidden_dropout_prob						)
											lowerCAmelCase   =		nn.Linear(config.hidden_size       , self.config.num_labels						)






				@add_start_docstrings_to_model_forward(UpperCAmelCase_						)
				def       __snake_case		( self       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=-1       , UpperCAmelCase_=False       , ):
											lowerCAmelCase   =		self.num_layers
											try:
																		lowerCAmelCase   =		self.roberta(
																		    UpperCAmelCase_       , attention_mask=UpperCAmelCase_       , token_type_ids=UpperCAmelCase_       , position_ids=UpperCAmelCase_       , head_mask=UpperCAmelCase_       , inputs_embeds=UpperCAmelCase_       , )

																		lowerCAmelCase   =		outputs[1]

																		lowerCAmelCase   =		self.dropout(UpperCAmelCase_						)
																		lowerCAmelCase   =		self.classifier(UpperCAmelCase_						)
																		lowerCAmelCase   =		(logits,) + outputs[2:]  # add hidden states and attention if they are here
											except HighwayException as e:
																		lowerCAmelCase   =		e.message
																		lowerCAmelCase   =		e.exit_layer
																		lowerCAmelCase   =		outputs[0]

											if not self.training:
																		lowerCAmelCase   =		entropy(UpperCAmelCase_						)
																		lowerCAmelCase   =		[]
																		lowerCAmelCase   =		[]
											if labels is not None:
																		if self.num_labels == 1:
																									#  We are doing regression
																									lowerCAmelCase   =		MSELoss()
																									lowerCAmelCase   =		loss_fct(logits.view(-1						)       , labels.view(-1						)						)
																		else:
																									lowerCAmelCase   =		CrossEntropyLoss()
																									lowerCAmelCase   =		loss_fct(logits.view(-1       , self.num_labels						)       , labels.view(-1						)						)

																		# work with highway exits
																		lowerCAmelCase   =		[]
																		for highway_exit in outputs[-1]:
																									lowerCAmelCase   =		highway_exit[0]
																									if not self.training:
																																highway_logits_all.append(UpperCAmelCase_						)
																																highway_entropy.append(highway_exit[2]						)
																									if self.num_labels == 1:
																																#  We are doing regression
																																lowerCAmelCase   =		MSELoss()
																																lowerCAmelCase   =		loss_fct(highway_logits.view(-1						)       , labels.view(-1						)						)
																									else:
																																lowerCAmelCase   =		CrossEntropyLoss()
																																lowerCAmelCase   =		loss_fct(highway_logits.view(-1       , self.num_labels						)       , labels.view(-1						)						)
																									highway_losses.append(UpperCAmelCase_						)

																		if train_highway:
																									lowerCAmelCase   =		(sum(highway_losses[:-1]						),) + outputs
																									# exclude the final highway, of course
																		else:
																									lowerCAmelCase   =		(loss,) + outputs
											if not self.training:
																		lowerCAmelCase   =		outputs + ((original_entropy, highway_entropy), exit_layer)
																		if output_layer >= 0:
																									lowerCAmelCase   =		(
																									    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																									)  # use the highway of the last layer

											return outputs  # (loss), logits, (hidden_states), (attentions), entropy









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


UpperCAmelCase_						={
    """configuration_efficientformer""": [
        """EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """EfficientFormerConfig""",
    ]
}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_						=["""EfficientFormerImageProcessor"""]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_						=[
		    """EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """EfficientFormerForImageClassification""",
		    """EfficientFormerForImageClassificationWithTeacher""",
		    """EfficientFormerModel""",
		    """EfficientFormerPreTrainedModel""",
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase_						=[
		    """TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
		    """TFEfficientFormerForImageClassification""",
		    """TFEfficientFormerForImageClassificationWithTeacher""",
		    """TFEfficientFormerModel""",
		    """TFEfficientFormerPreTrainedModel""",
		]

if TYPE_CHECKING:
		from .configuration_efficientformer import EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, EfficientFormerConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .image_processing_efficientformer import EfficientFormerImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_efficientformer import (
				    EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    EfficientFormerForImageClassification,
				    EfficientFormerForImageClassificationWithTeacher,
				    EfficientFormerModel,
				    EfficientFormerPreTrainedModel,
				)
		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_efficientformer import (
				    TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFEfficientFormerForImageClassification,
				    TFEfficientFormerForImageClassificationWithTeacher,
				    TFEfficientFormerModel,
				    TFEfficientFormerPreTrainedModel,
				)

else:
		import sys

		UpperCAmelCase_						=_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



