
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy




@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''	,							UpperCamelCase_	,							)
class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  RobertaConfig
     SCREAMING_SNAKE_CASE_		   =  '''roberta'''






     def __init__(							self						,						__SCREAMING_SNAKE_CASE  )  ->   Dict:




           """simple docstring"""







           super().__init__(__SCREAMING_SNAKE_CASE  )

           UpperCamelCase__    :   Union[str, Any]					=				RobertaEmbeddings(__SCREAMING_SNAKE_CASE  )
           self.init_weights()




@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''	,							UpperCamelCase_	,							)
class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  RobertaConfig
     SCREAMING_SNAKE_CASE_		   =  '''roberta'''






     def __init__(							self						,						__SCREAMING_SNAKE_CASE  )  ->   Dict:




           """simple docstring"""







           super().__init__(__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Any					=				config.num_labels
           UpperCamelCase__    :   List[str]					=				config.num_hidden_layers

           UpperCamelCase__    :   Tuple					=				DeeRobertaModel(__SCREAMING_SNAKE_CASE  )
           UpperCamelCase__    :   Any					=				nn.Dropout(config.hidden_dropout_prob  )
           UpperCamelCase__    :   Union[str, Any]					=				nn.Linear(config.hidden_size						,						self.config.num_labels  )






     @add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE  )
     def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=-1						,						__SCREAMING_SNAKE_CASE=False						,						)  ->   List[str]:




           """simple docstring"""







           UpperCamelCase__    :   Union[str, Any]					=				self.num_layers
           try:
                 UpperCamelCase__    :   Tuple					=				self.roberta(
                     __SCREAMING_SNAKE_CASE						,						attention_mask=__SCREAMING_SNAKE_CASE						,						token_type_ids=__SCREAMING_SNAKE_CASE						,						position_ids=__SCREAMING_SNAKE_CASE						,						head_mask=__SCREAMING_SNAKE_CASE						,						inputs_embeds=__SCREAMING_SNAKE_CASE						,						)

                 UpperCamelCase__    :   Union[str, Any]					=				outputs[1]

                 UpperCamelCase__    :   Union[str, Any]					=				self.dropout(__SCREAMING_SNAKE_CASE  )
                 UpperCamelCase__    :   List[str]					=				self.classifier(__SCREAMING_SNAKE_CASE  )
                 UpperCamelCase__    :   Tuple					=				(logits,) + outputs[2:]  # add hidden states and attention if they are here
           except HighwayException as e:
                 UpperCamelCase__    :   Optional[Any]					=				e.message
                 UpperCamelCase__    :   Optional[Any]					=				e.exit_layer
                 UpperCamelCase__    :   Union[str, Any]					=				outputs[0]

           if not self.training:
                 UpperCamelCase__    :   List[Any]					=				entropy(__SCREAMING_SNAKE_CASE  )
                 UpperCamelCase__    :   List[Any]					=				[]
                 UpperCamelCase__    :   Tuple					=				[]
           if labels is not None:
                 if self.num_labels == 1:
                       #  We are doing regression
                       UpperCamelCase__    :   List[Any]					=				MSELoss()
                       UpperCamelCase__    :   Tuple					=				loss_fct(logits.view(-1  )						,						labels.view(-1  )  )
                 else:
                       UpperCamelCase__    :   List[Any]					=				CrossEntropyLoss()
                       UpperCamelCase__    :   int					=				loss_fct(logits.view(-1						,						self.num_labels  )						,						labels.view(-1  )  )

                 # work with highway exits
                 UpperCamelCase__    :   List[str]					=				[]
                 for highway_exit in outputs[-1]:
                       UpperCamelCase__    :   str					=				highway_exit[0]
                       if not self.training:
                             highway_logits_all.append(__SCREAMING_SNAKE_CASE  )
                             highway_entropy.append(highway_exit[2]  )
                       if self.num_labels == 1:
                             #  We are doing regression
                             UpperCamelCase__    :   Union[str, Any]					=				MSELoss()
                             UpperCamelCase__    :   List[Any]					=				loss_fct(highway_logits.view(-1  )						,						labels.view(-1  )  )
                       else:
                             UpperCamelCase__    :   List[Any]					=				CrossEntropyLoss()
                             UpperCamelCase__    :   int					=				loss_fct(highway_logits.view(-1						,						self.num_labels  )						,						labels.view(-1  )  )
                       highway_losses.append(__SCREAMING_SNAKE_CASE  )

                 if train_highway:
                       UpperCamelCase__    :   Union[str, Any]					=				(sum(highway_losses[:-1]  ),) + outputs
                       # exclude the final highway, of course
                 else:
                       UpperCamelCase__    :   Union[str, Any]					=				(loss,) + outputs
           if not self.training:
                 UpperCamelCase__    :   int					=				outputs + ((original_entropy, highway_entropy), exit_layer)
                 if output_layer >= 0:
                       UpperCamelCase__    :   List[str]					=				(
                           (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                       )  # use the highway of the last layer

           return outputs  # (loss), logits, (hidden_states), (attentions), entropy




lowerCamelCase      ={
    "Pillow": "Pillow",
    "accelerate": "accelerate>=0.11.0",
    "compel": "compel==0.1.8",
    "black": "black~=23.1",
    "datasets": "datasets",
    "filelock": "filelock",
    "flax": "flax>=0.4.1",
    "hf-doc-builder": "hf-doc-builder>=0.3.0",
    "huggingface-hub": "huggingface-hub>=0.13.2",
    "requests-mock": "requests-mock==1.10.0",
    "importlib_metadata": "importlib_metadata",
    "invisible-watermark": "invisible-watermark",
    "isort": "isort>=5.5.4",
    "jax": "jax>=0.2.8,!=0.3.2",
    "jaxlib": "jaxlib>=0.1.65",
    "Jinja2": "Jinja2",
    "k-diffusion": "k-diffusion>=0.0.12",
    "torchsde": "torchsde",
    "note_seq": "note_seq",
    "librosa": "librosa",
    "numpy": "numpy",
    "omegaconf": "omegaconf",
    "parameterized": "parameterized",
    "protobuf": "protobuf>=3.20.3,<4",
    "pytest": "pytest",
    "pytest-timeout": "pytest-timeout",
    "pytest-xdist": "pytest-xdist",
    "ruff": "ruff>=0.0.241",
    "safetensors": "safetensors",
    "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92",
    "scipy": "scipy",
    "onnx": "onnx",
    "regex": "regex!=2019.12.17",
    "requests": "requests",
    "tensorboard": "tensorboard",
    "torch": "torch>=1.4",
    "torchvision": "torchvision",
    "transformers": "transformers>=4.25.1",
    "urllib3": "urllib3<=2.0.0",
}


