




import unittest

from transformers import DebertaVaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         DebertaVaForMaskedLM,
         DebertaVaForMultipleChoice,
         DebertaVaForQuestionAnswering,
         DebertaVaForSequenceClassification,
         DebertaVaForTokenClassification,
         DebertaVaModel,
     )
     from transformers.models.deberta_va.modeling_deberta_va import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST






class  _lowerCAmelCase     (  _lowercase	):

      def __init__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase=13      ,				__UpperCAmelCase=7      ,				__UpperCAmelCase=True      ,				__UpperCAmelCase=True      ,				__UpperCAmelCase=True      ,				__UpperCAmelCase=True      ,				__UpperCAmelCase=99      ,				__UpperCAmelCase=32      ,				__UpperCAmelCase=5      ,				__UpperCAmelCase=4      ,				__UpperCAmelCase=37      ,				__UpperCAmelCase="gelu"      ,				__UpperCAmelCase=0.1      ,				__UpperCAmelCase=0.1      ,				__UpperCAmelCase=512      ,				__UpperCAmelCase=16      ,				__UpperCAmelCase=2      ,				__UpperCAmelCase=0.02      ,				__UpperCAmelCase=False      ,				__UpperCAmelCase=True      ,				__UpperCAmelCase="None"      ,				__UpperCAmelCase=3      ,				__UpperCAmelCase=4      ,				__UpperCAmelCase=None      ,				):
        lowerCAmelCase__				:  str		       =     parent
        lowerCAmelCase__				:  List[Any]		       =     batch_size
        lowerCAmelCase__				:  int		       =     seq_length
        lowerCAmelCase__				:  List[Any]		       =     is_training
        lowerCAmelCase__				:  Any		       =     use_input_mask
        lowerCAmelCase__				:  str		       =     use_token_type_ids
        lowerCAmelCase__				:  Dict		       =     use_labels
        lowerCAmelCase__				:  Optional[Any]		       =     vocab_size
        lowerCAmelCase__				:  List[Any]		       =     hidden_size
        lowerCAmelCase__				:  Any		       =     num_hidden_layers
        lowerCAmelCase__				:  Optional[int]		       =     num_attention_heads
        lowerCAmelCase__				:  Optional[Any]		       =     intermediate_size
        lowerCAmelCase__				:  str		       =     hidden_act
        lowerCAmelCase__				:  int		       =     hidden_dropout_prob
        lowerCAmelCase__				:  str		       =     attention_probs_dropout_prob
        lowerCAmelCase__				:  List[Any]		       =     max_position_embeddings
        lowerCAmelCase__				:  Tuple		       =     type_vocab_size
        lowerCAmelCase__				:  List[str]		       =     type_sequence_label_size
        lowerCAmelCase__				:  List[str]		       =     initializer_range
        lowerCAmelCase__				:  Union[str, Any]		       =     num_labels
        lowerCAmelCase__				:  List[Any]		       =     num_choices
        lowerCAmelCase__				:  Optional[Any]		       =     relative_attention
        lowerCAmelCase__				:  Any		       =     position_biased_input
        lowerCAmelCase__				:  Union[str, Any]		       =     pos_att_type
        lowerCAmelCase__				:  Any		       =     scope

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Union[str, Any]		       =     ids_tensor([self.batch_size, self.seq_length]      ,				self.vocab_size		)

        lowerCAmelCase__				:  List[Any]		       =     None
        if self.use_input_mask:
          lowerCAmelCase__				:  Optional[Any]		       =     ids_tensor([self.batch_size, self.seq_length]      ,				vocab_size=2		)

        lowerCAmelCase__				:  Dict		       =     None
        if self.use_token_type_ids:
          lowerCAmelCase__				:  int		       =     ids_tensor([self.batch_size, self.seq_length]      ,				self.type_vocab_size		)

        lowerCAmelCase__				:  Dict		       =     None
        lowerCAmelCase__				:  Union[str, Any]		       =     None
        lowerCAmelCase__				:  Dict		       =     None
        if self.use_labels:
          lowerCAmelCase__				:  Optional[int]		       =     ids_tensor([self.batch_size]      ,				self.type_sequence_label_size		)
          lowerCAmelCase__				:  int		       =     ids_tensor([self.batch_size, self.seq_length]      ,				self.num_labels		)
          lowerCAmelCase__				:  Dict		       =     ids_tensor([self.batch_size]      ,				self.num_choices		)

        lowerCAmelCase__				:  Optional[Any]		       =     self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

      def 	__magic_name__(   self		):
        return DebertaVaConfig(
            vocab_size=self.vocab_size      ,				hidden_size=self.hidden_size      ,				num_hidden_layers=self.num_hidden_layers      ,				num_attention_heads=self.num_attention_heads      ,				intermediate_size=self.intermediate_size      ,				hidden_act=self.hidden_act      ,				hidden_dropout_prob=self.hidden_dropout_prob      ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,				max_position_embeddings=self.max_position_embeddings      ,				type_vocab_size=self.type_vocab_size      ,				initializer_range=self.initializer_range      ,				relative_attention=self.relative_attention      ,				position_biased_input=self.position_biased_input      ,				pos_att_type=self.pos_att_type      ,				)

      def 	__magic_name__(   self      ,				__UpperCAmelCase		):
        self.parent.assertListEqual(list(result.loss.size()		)      ,				[]		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  Optional[int]		       =     DebertaVaModel(config=__UpperCAmelCase		)
        model.to(__UpperCAmelCase		)
        model.eval()
        lowerCAmelCase__				:  Optional[Any]		       =     model(__UpperCAmelCase      ,				attention_mask=__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase		)[0]
        lowerCAmelCase__				:  List[Any]		       =     model(__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase		)[0]
        lowerCAmelCase__				:  int		       =     model(__UpperCAmelCase		)[0]

        self.parent.assertListEqual(list(sequence_output.size()		)      ,				[self.batch_size, self.seq_length, self.hidden_size]		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  List[Any]		       =     DebertaVaForMaskedLM(config=__UpperCAmelCase		)
        model.to(__UpperCAmelCase		)
        model.eval()
        lowerCAmelCase__				:  str		       =     model(__UpperCAmelCase      ,				attention_mask=__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase      ,				labels=__UpperCAmelCase		)

        self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.seq_length, self.vocab_size)		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  int		       =     self.num_labels
        lowerCAmelCase__				:  int		       =     DebertaVaForSequenceClassification(__UpperCAmelCase		)
        model.to(__UpperCAmelCase		)
        model.eval()
        lowerCAmelCase__				:  Union[str, Any]		       =     model(__UpperCAmelCase      ,				attention_mask=__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase      ,				labels=__UpperCAmelCase		)
        self.parent.assertListEqual(list(result.logits.size()		)      ,				[self.batch_size, self.num_labels]		)
        self.check_loss_output(__UpperCAmelCase		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  Tuple		       =     self.num_labels
        lowerCAmelCase__				:  str		       =     DebertaVaForTokenClassification(config=__UpperCAmelCase		)
        model.to(__UpperCAmelCase		)
        model.eval()
        lowerCAmelCase__				:  Tuple		       =     model(__UpperCAmelCase      ,				attention_mask=__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase      ,				labels=__UpperCAmelCase		)
        self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.seq_length, self.num_labels)		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  Any		       =     DebertaVaForQuestionAnswering(config=__UpperCAmelCase		)
        model.to(__UpperCAmelCase		)
        model.eval()
        lowerCAmelCase__				:  str		       =     model(
            __UpperCAmelCase      ,				attention_mask=__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase      ,				start_positions=__UpperCAmelCase      ,				end_positions=__UpperCAmelCase      ,				)
        self.parent.assertEqual(result.start_logits.shape      ,				(self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape      ,				(self.batch_size, self.seq_length)		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  str		       =     DebertaVaForMultipleChoice(config=__UpperCAmelCase		)
        model.to(__UpperCAmelCase		)
        model.eval()
        lowerCAmelCase__				:  Optional[Any]		       =     input_ids.unsqueeze(1		).expand(-1      ,				self.num_choices      ,				-1		).contiguous()
        lowerCAmelCase__				:  Any		       =     token_type_ids.unsqueeze(1		).expand(-1      ,				self.num_choices      ,				-1		).contiguous()
        lowerCAmelCase__				:  List[Any]		       =     input_mask.unsqueeze(1		).expand(-1      ,				self.num_choices      ,				-1		).contiguous()
        lowerCAmelCase__				:  int		       =     model(
            __UpperCAmelCase      ,				attention_mask=__UpperCAmelCase      ,				token_type_ids=__UpperCAmelCase      ,				labels=__UpperCAmelCase      ,				)
        self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.num_choices)		)







      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Union[str, Any]		       =     self.prepare_config_and_inputs()
        (
            (
            lowerCAmelCase__
        )       ,			(
            lowerCAmelCase__
        )       ,			(
            lowerCAmelCase__
        )       ,			(
            lowerCAmelCase__
        )       ,			(
            lowerCAmelCase__
        )       ,			(
            lowerCAmelCase__
        )       ,			(
            lowerCAmelCase__
        )       ,			
        )				:  Tuple		       =     config_and_inputs
        lowerCAmelCase__				:  Optional[int]		       =     {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict






@require_torch
class  _lowerCAmelCase     (  _lowercase    ,			_lowercase    ,			unittest.TestCase	):
      A__		     =		(
          (
              DebertaVaModel,
              DebertaVaForMaskedLM,
              DebertaVaForSequenceClassification,
              DebertaVaForTokenClassification,
              DebertaVaForQuestionAnswering,
              DebertaVaForMultipleChoice,
          )
          if is_torch_available()
          else ()
      )
      A__		     =		(
          {
              'feature-extraction': DebertaVaModel,
              'fill-mask': DebertaVaForMaskedLM,
              'question-answering': DebertaVaForQuestionAnswering,
              'text-classification': DebertaVaForSequenceClassification,
              'token-classification': DebertaVaForTokenClassification,
              'zero-shot': DebertaVaForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )

      A__		     =		True
      A__		     =		False
      A__		     =		False
      A__		     =		False
      A__		     =		False

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  str		       =     DebertaVaModelTester(self		)
        lowerCAmelCase__				:  Union[str, Any]		       =     ConfigTester(self      ,				config_class=__UpperCAmelCase      ,				hidden_size=37		)

      def 	__magic_name__(   self		):
        self.config_tester.run_common_tests()

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Dict		       =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_model(*__UpperCAmelCase		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Optional[Any]		       =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_sequence_classification(*__UpperCAmelCase		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  List[Any]		       =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_masked_lm(*__UpperCAmelCase		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  List[Any]		       =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_question_answering(*__UpperCAmelCase		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Tuple		       =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_token_classification(*__UpperCAmelCase		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  str		       =     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deberta_for_multiple_choice(*__UpperCAmelCase		)







      @slow
      def 	__magic_name__(   self		):
        for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase__				:  List[Any]		       =     DebertaVaModel.from_pretrained(__UpperCAmelCase		)
          self.assertIsNotNone(__UpperCAmelCase		)






@require_torch
@require_sentencepiece
@require_tokenizers
class  _lowerCAmelCase     (  unittest.TestCase	):

      @unittest.skip(reason='''Model not available yet'''		)
      def 	__magic_name__(   self		):
        pass







      @slow
      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  str		       =     DebertaVaModel.from_pretrained('''microsoft/deberta-v2-xlarge'''		)

        lowerCAmelCase__				:  List[Any]		       =     torch.tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]		)
        lowerCAmelCase__				:  int		       =     torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
        with torch.no_grad():
          lowerCAmelCase__				:  str		       =     model(__UpperCAmelCase      ,				attention_mask=__UpperCAmelCase		)[0]
        # compare the actual values for a slice.
        lowerCAmelCase__				:  Tuple		       =     torch.tensor(
            [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]]		)
        self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,				__UpperCAmelCase      ,				atol=1e-4		)      ,				f"""{output[:, 1:4, 1:4]}"""		)







from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
from .unet_ad_blocks import get_down_block, get_mid_block, get_out_block, get_up_block






@dataclass
class  _lowerCAmelCase     (  _lowercase	):
      A__		     =		42






class  _lowerCAmelCase     (  _lowercase    ,			_lowercase	):

      @register_to_config
      def __init__(   self      ,				__UpperCAmelCase = 6_5536      ,				__UpperCAmelCase = None      ,				__UpperCAmelCase = 2      ,				__UpperCAmelCase = 2      ,				__UpperCAmelCase = 0      ,				__UpperCAmelCase = "fourier"      ,				__UpperCAmelCase = True      ,				__UpperCAmelCase = False      ,				__UpperCAmelCase = 0.0      ,				__UpperCAmelCase = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D")      ,				__UpperCAmelCase = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip")      ,				__UpperCAmelCase = "UNetMidBlock1D"      ,				__UpperCAmelCase = None      ,				__UpperCAmelCase = (32, 32, 64)      ,				__UpperCAmelCase = None      ,				__UpperCAmelCase = 8      ,				__UpperCAmelCase = 1      ,				__UpperCAmelCase = False      ,				):
        super().__init__()
        lowerCAmelCase__				:  Dict		       =     sample_size

        # time
        if time_embedding_type == "fourier":
          lowerCAmelCase__				:  str		       =     GaussianFourierProjection(
              embedding_size=8      ,				set_W_to_weight=__UpperCAmelCase      ,				log=__UpperCAmelCase      ,				flip_sin_to_cos=__UpperCAmelCase		)
          lowerCAmelCase__				:  int		       =     2 * block_out_channels[0]
        elif time_embedding_type == "positional":
          lowerCAmelCase__				:  int		       =     Timesteps(
              block_out_channels[0]      ,				flip_sin_to_cos=__UpperCAmelCase      ,				downscale_freq_shift=__UpperCAmelCase		)
          lowerCAmelCase__				:  str		       =     block_out_channels[0]

        if use_timestep_embedding:
          lowerCAmelCase__				:  str		       =     block_out_channels[0] * 4
          lowerCAmelCase__				:  Dict		       =     TimestepEmbedding(
              in_channels=__UpperCAmelCase      ,				time_embed_dim=__UpperCAmelCase      ,				act_fn=__UpperCAmelCase      ,				out_dim=block_out_channels[0]      ,				)

        lowerCAmelCase__				:  str		       =     nn.ModuleList([]		)
        lowerCAmelCase__				:  Optional[Any]		       =     None
        lowerCAmelCase__				:  Optional[int]		       =     nn.ModuleList([]		)
        lowerCAmelCase__				:  Optional[Any]		       =     None

        # down
        lowerCAmelCase__				:  List[Any]		       =     in_channels
        for i, down_block_type in enumerate(__UpperCAmelCase		):
          lowerCAmelCase__				:  List[str]		       =     output_channel
          lowerCAmelCase__				:  Any		       =     block_out_channels[i]

          if i == 0:
            input_channel += extra_in_channels

          lowerCAmelCase__				:  Any		       =     i == len(__UpperCAmelCase		) - 1

          lowerCAmelCase__				:  Any		       =     get_down_block(
              __UpperCAmelCase      ,				num_layers=__UpperCAmelCase      ,				in_channels=__UpperCAmelCase      ,				out_channels=__UpperCAmelCase      ,				temb_channels=block_out_channels[0]      ,				add_downsample=not is_final_block or downsample_each_block      ,				)
          self.down_blocks.append(__UpperCAmelCase		)

        # mid
        lowerCAmelCase__				:  Optional[int]		       =     get_mid_block(
            __UpperCAmelCase      ,				in_channels=block_out_channels[-1]      ,				mid_channels=block_out_channels[-1]      ,				out_channels=block_out_channels[-1]      ,				embed_dim=block_out_channels[0]      ,				num_layers=__UpperCAmelCase      ,				add_downsample=__UpperCAmelCase      ,				)

        # up
        lowerCAmelCase__				:  Union[str, Any]		       =     list(reversed(__UpperCAmelCase		)		)
        lowerCAmelCase__				:  List[str]		       =     reversed_block_out_channels[0]
        if out_block_type is None:
          lowerCAmelCase__				:  Optional[int]		       =     out_channels
        else:
          lowerCAmelCase__				:  int		       =     block_out_channels[0]

        for i, up_block_type in enumerate(__UpperCAmelCase		):
          lowerCAmelCase__				:  Optional[int]		       =     output_channel
          lowerCAmelCase__				:  Any		       =     (
              reversed_block_out_channels[i + 1] if i < len(__UpperCAmelCase		) - 1 else final_upsample_channels
          )

          lowerCAmelCase__				:  Optional[int]		       =     i == len(__UpperCAmelCase		) - 1

          lowerCAmelCase__				:  Optional[int]		       =     get_up_block(
              __UpperCAmelCase      ,				num_layers=__UpperCAmelCase      ,				in_channels=__UpperCAmelCase      ,				out_channels=__UpperCAmelCase      ,				temb_channels=block_out_channels[0]      ,				add_upsample=not is_final_block      ,				)
          self.up_blocks.append(__UpperCAmelCase		)
          lowerCAmelCase__				:  int		       =     output_channel

        # out
        lowerCAmelCase__				:  Union[str, Any]		       =     norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4      ,				32		)
        lowerCAmelCase__				:  Dict		       =     get_out_block(
            out_block_type=__UpperCAmelCase      ,				num_groups_out=__UpperCAmelCase      ,				embed_dim=block_out_channels[0]      ,				out_channels=__UpperCAmelCase      ,				act_fn=__UpperCAmelCase      ,				fc_dim=block_out_channels[-1] // 4      ,				)







      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase      ,				__UpperCAmelCase = True      ,				):
        lowerCAmelCase__				:  int		       =     timestep
        if not torch.is_tensor(__UpperCAmelCase		):
          lowerCAmelCase__				:  Any		       =     torch.tensor([timesteps]      ,				dtype=torch.long      ,				device=sample.device		)
        elif torch.is_tensor(__UpperCAmelCase		) and len(timesteps.shape		) == 0:
          lowerCAmelCase__				:  Any		       =     timesteps[None].to(sample.device		)

        lowerCAmelCase__				:  int		       =     self.time_proj(__UpperCAmelCase		)
        if self.config.use_timestep_embedding:
          lowerCAmelCase__				:  str		       =     self.time_mlp(__UpperCAmelCase		)
        else:
          lowerCAmelCase__				:  Tuple		       =     timestep_embed[..., None]
          lowerCAmelCase__				:  Union[str, Any]		       =     timestep_embed.repeat([1, 1, sample.shape[2]]		).to(sample.dtype		)
          lowerCAmelCase__				:  Any		       =     timestep_embed.broadcast_to((sample.shape[:1] + timestep_embed.shape[1:])		)

        # 2. down
        lowerCAmelCase__				:  int		       =     ()
        for downsample_block in self.down_blocks:
          lowerCAmelCase__       ,			lowerCAmelCase__				:  int		       =     downsample_block(hidden_states=__UpperCAmelCase      ,				temb=__UpperCAmelCase		)
          down_block_res_samples += res_samples

        # 3. mid
        if self.mid_block:
          lowerCAmelCase__				:  Dict		       =     self.mid_block(__UpperCAmelCase      ,				__UpperCAmelCase		)

        # 4. up
        for i, upsample_block in enumerate(self.up_blocks		):
          lowerCAmelCase__				:  Tuple		       =     down_block_res_samples[-1:]
          lowerCAmelCase__				:  int		       =     down_block_res_samples[:-1]
          lowerCAmelCase__				:  Optional[int]		       =     upsample_block(__UpperCAmelCase      ,				res_hidden_states_tuple=__UpperCAmelCase      ,				temb=__UpperCAmelCase		)

        # 5. post-process
        if self.out_block:
          lowerCAmelCase__				:  Any		       =     self.out_block(__UpperCAmelCase      ,				__UpperCAmelCase		)

        if not return_dict:
          return (sample,)

        return UNetaDOutput(sample=__UpperCAmelCase		)

