





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy

@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''			,       _lowerCamelCase			,       )
class    _UpperCAmelCase							(		_lowerCamelCase     ):
      a						       =							RobertaConfig
      a						       =							'''roberta'''
      def __init__(		self      ,       a__       ):
         super().__init__(a__       )

         A_     :				Optional[Any]			      =	RobertaEmbeddings(a__       )
         self.init_weights()






@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''			,       _lowerCamelCase			,       )
class    _UpperCAmelCase							(		_lowerCamelCase     ):
      a						       =							RobertaConfig
      a						       =							'''roberta'''
      def __init__(		self      ,       a__       ):
         super().__init__(a__       )
         A_     :				Union[str, Any]			      =	config.num_labels
         A_     :				List[str]			      =	config.num_hidden_layers

         A_     :				Union[str, Any]			      =	DeeRobertaModel(a__       )
         A_     :				Optional[int]			      =	nn.Dropout(config.hidden_dropout_prob       )
         A_     :				List[Any]			      =	nn.Linear(config.hidden_size      ,       self.config.num_labels       )






      @add_start_docstrings_to_model_forward(a__       )
      def 		_lowerCamelCase      (		self      ,       a__=None      ,       a__=None      ,       a__=None      ,       a__=None      ,       a__=None      ,       a__=None      ,       a__=None      ,       a__=-1      ,       a__=False      ,       ):
         A_     :				int			      =	self.num_layers
         try:
            A_     :				Any			      =	self.roberta(
                a__      ,       attention_mask=a__      ,       token_type_ids=a__      ,       position_ids=a__      ,       head_mask=a__      ,       inputs_embeds=a__      ,       )

            A_     :				int			      =	outputs[1]

            A_     :				Any			      =	self.dropout(a__       )
            A_     :				List[str]			      =	self.classifier(a__       )
            A_     :				List[str]			      =	(logits,) + outputs[2:]  # add hidden states and attention if they are here
         except HighwayException as e:
            A_     :				Any			      =	e.message
            A_     :				Union[str, Any]			      =	e.exit_layer
            A_     :				Any			      =	outputs[0]

         if not self.training:
            A_     :				Optional[Any]			      =	entropy(a__       )
            A_     :				Optional[Any]			      =	[]
            A_     :				int			      =	[]
         if labels is not None:
            if self.num_labels == 1:
               #  We are doing regression
               A_     :				Any			      =	MSELoss()
               A_     :				Any			      =	loss_fct(logits.view(-1       )      ,       labels.view(-1       )       )
            else:
               A_     :				Union[str, Any]			      =	CrossEntropyLoss()
               A_     :				Tuple			      =	loss_fct(logits.view(-1      ,       self.num_labels       )      ,       labels.view(-1       )       )

            # work with highway exits
            A_     :				Union[str, Any]			      =	[]
            for highway_exit in outputs[-1]:
               A_     :				Optional[int]			      =	highway_exit[0]
               if not self.training:
                  highway_logits_all.append(a__       )
                  highway_entropy.append(highway_exit[2]       )
               if self.num_labels == 1:
                  #  We are doing regression
                  A_     :				List[Any]			      =	MSELoss()
                  A_     :				int			      =	loss_fct(highway_logits.view(-1       )      ,       labels.view(-1       )       )
               else:
                  A_     :				Dict			      =	CrossEntropyLoss()
                  A_     :				Optional[int]			      =	loss_fct(highway_logits.view(-1      ,       self.num_labels       )      ,       labels.view(-1       )       )
               highway_losses.append(a__       )

            if train_highway:
               A_     :				int			      =	(sum(highway_losses[:-1]       ),) + outputs
               # exclude the final highway, of course
            else:
               A_     :				List[Any]			      =	(loss,) + outputs
         if not self.training:
            A_     :				List[Any]			      =	outputs + ((original_entropy, highway_entropy), exit_layer)
            if output_layer >= 0:
               A_     :				Dict			      =	(
                   (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
               )  # use the highway of the last layer

         return outputs  # (loss), logits, (hidden_states), (attentions), entropy













import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler

class    _UpperCAmelCase							:
      def __init__(		self      ,       a__       ):
         if isinstance(a__      ,       a__       ):
            # Don't modify user's data should they want to reuse it (e.g. in tests), because once we
            # modified it, it will not be accepted here again, since `auto` values would have been overridden
            A_     :				Optional[Any]			      =	deepcopy(a__       )
         elif os.path.exists(a__       ):
            with io.open(a__      ,       """r"""      ,       encoding="""utf-8"""       ) as f:
               A_     :				str			      =	json.load(a__       )
         else:
            try:
               A_     :				Dict			      =	baseaa.urlsafe_baadecode(a__       ).decode("""utf-8"""       )
               A_     :				List[Any]			      =	json.loads(a__       )
            except (UnicodeDecodeError, AttributeError, ValueError):
               raise ValueError(
                   F"""Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}"""       )

         A_     :				Any			      =	config

         self.set_stage_and_offload()
      def 		_lowerCamelCase      (		self       ):
         # zero stage - this is done as early as possible, before model is created, to allow
         # ``is_deepspeed_zero3_enabled`` query and getting to the early deepspeed config object
         # during ``zero.Init()`` which needs to know the dtype, and some other hparams.
         A_     :				List[str]			      =	self.get_value("""zero_optimization.stage"""      ,       -1       )

         # offload
         A_     :				Any			      =	False
         if self.is_zeroa() or self.is_zeroa():
            A_     :				Optional[int]			      =	set(["""cpu""", """nvme"""]       )
            A_     :				Dict			      =	set(
                [
                    self.get_value("""zero_optimization.offload_optimizer.device"""       ),
                    self.get_value("""zero_optimization.offload_param.device"""       ),
                ]       )
            if len(offload_devices & offload_devices_valid       ) > 0:
               A_     :				Tuple			      =	True
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				List[Any]			      =	self.config

         # find the config node of interest if it exists
         A_     :				Optional[Any]			      =	ds_key_long.split("""."""       )
         A_     :				Union[str, Any]			      =	nodes.pop()
         for node in nodes:
            A_     :				List[str]			      =	config.get(a__       )
            if config is None:
               return None, ds_key

         return config, ds_key
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=None       ):
         A_      ,				A_     :				Union[str, Any]			      =	self.find_config_node(a__       )
         if config is None:
            return default
         return config.get(a__      ,       a__       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=False       ):
         A_     :				Union[str, Any]			      =	self.config

         # find the config node of interest if it exists
         A_     :				str			      =	ds_key_long.split("""."""       )
         for node in nodes:
            A_     :				int			      =	config
            A_     :				int			      =	config.get(a__       )
            if config is None:
               if must_exist:
                  raise ValueError(F"""Can't find {ds_key_long} entry in the config: {self.config}"""       )
               else:
                  return

        # if found remove it
         if parent_config is not None:
            parent_config.pop(a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Optional[Any]			      =	self.get_value(a__       )
         return False if value is None else bool(a__       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Optional[Any]			      =	self.get_value(a__       )
         return False if value is None else not bool(a__       )
      def 		_lowerCamelCase      (		self       ):
         return self._stage == 2
      def 		_lowerCamelCase      (		self       ):
         return self._stage == 3






      def 		_lowerCamelCase      (		self       ):
         return self._offload






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__       ):
         A_     :				Any			      =	engine






      def 		_lowerCamelCase      (		self      ,       a__      ,       **a__       ):
         # runs backpropagation and handles mixed precision
         self.engine.backward(a__      ,       **a__       )

         # Deepspeed's `engine.step` performs the following operations:
         # - gradient accumulation check
         # - gradient clipping
         # - optimizer step
         # - zero grad
         # - checking overflow
         # - lr_scheduler step (only if engine.lr_scheduler is not None)
         self.engine.step()
         # and this plugin overrides the above calls with no-ops when Accelerate runs under
         # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
         # training loop that works transparently under many training regimes.






class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__       ):
         super().__init__(a__      ,       device_placement=a__      ,       scaler=a__       )
         A_     :				Dict			      =	hasattr(self.optimizer      ,       """overflow"""       )
      def 		_lowerCamelCase      (		self      ,       a__=None       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed
      def 		_lowerCamelCase      (		self       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






      @property
      def 		_lowerCamelCase      (		self       ):
         if self.__has_overflow__:
            return self.optimizer.overflow
         return False






class    _UpperCAmelCase							(		_lowerCamelCase     ):
      def __init__(		self      ,       a__      ,       a__       ):
         super().__init__(a__      ,       a__       )






      def 		_lowerCamelCase      (		self       ):
         pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__=0.001      ,       a__=0      ,       **a__       ):
         A_     :				List[str]			      =	params
         A_     :				Any			      =	lr
         A_     :				int			      =	weight_decay
         A_     :				Optional[int]			      =	kwargs






class    _UpperCAmelCase							:
      def __init__(		self      ,       a__      ,       a__=None      ,       a__=0      ,       **a__       ):
         A_     :				Union[str, Any]			      =	optimizer
         A_     :				int			      =	total_num_steps
         A_     :				Any			      =	warmup_num_steps
         A_     :				int			      =	kwargs






