






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


UpperCamelCase_	:				List[Any]										= logging.get_logger(__name__)


UpperCamelCase_	:				Optional[Any]										= {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase_	:				List[Any]										= {
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json""",
    },
}

UpperCamelCase_	:				Optional[int]										= {
    """RUCAIBox/mvp""": 1024,
}


class      _lowercase (     lowerCAmelCase						):
      _a   :     Optional[int]								=   VOCAB_FILES_NAMES
      _a   :     Dict								=   PRETRAINED_VOCAB_FILES_MAP
      _a   :     Dict								=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _a   :     Any								=   ['''input_ids''', '''attention_mask''']
      _a   :     Union[str, Any]								=   MvpTokenizer


      def __init__(    self				:	Optional[Any]							,   a				:	Optional[Any]=None							,   a				:	List[Any]=None							,   a				:	int=None							,   a				:	Optional[Any]="replace"							,   a				:	Tuple="<s>"							,   a				:	Any="</s>"							,   a				:	Dict="</s>"							,   a				:	Optional[Any]="<s>"							,   a				:	str="<unk>"							,   a				:	Union[str, Any]="<pad>"							,   a				:	Optional[int]="<mask>"							,   a				:	str=False							,   a				:	Tuple=True							,   **a				:	str							,   ):





            """simple docstring"""







            super().__init__(
                a							,   a							,   tokenizer_file=a							,   errors=a							,   bos_token=a							,   eos_token=a							,   sep_token=a							,   cls_token=a							,   unk_token=a							,   pad_token=a							,   mask_token=a							,   add_prefix_space=a							,   trim_offsets=a							,   **a							,   )

            __snake_case      :    Optional[Any]  			=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
            if pre_tok_state.get('''add_prefix_space'''							,   a						) != add_prefix_space:
                  __snake_case      :    int  			=getattr(a							,   pre_tok_state.pop('''type'''						)						)
                  __snake_case      :    Optional[Any]  			=add_prefix_space
                  __snake_case      :    Union[str, Any]  			=pre_tok_class(**a						)

            __snake_case      :    Dict  			=add_prefix_space

            # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
            __snake_case      :    Union[str, Any]  			='''post_processor'''
            __snake_case      :    Optional[Any]  			=getattr(self.backend_tokenizer							,   a							,   a						)
            if tokenizer_component_instance:
                  __snake_case      :    Dict  			=json.loads(tokenizer_component_instance.__getstate__()						)

                  # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                  if "sep" in state:
                        __snake_case      :    Dict  			=tuple(state['''sep''']						)
                  if "cls" in state:
                        __snake_case      :    List[Any]  			=tuple(state['''cls''']						)

                  __snake_case      :    Optional[int]  			=False

                  if state.get('''add_prefix_space'''							,   a						) != add_prefix_space:
                        __snake_case      :    Dict  			=add_prefix_space
                        __snake_case      :    int  			=True

                  if state.get('''trim_offsets'''							,   a						) != trim_offsets:
                        __snake_case      :    List[Any]  			=trim_offsets
                        __snake_case      :    Optional[int]  			=True

                  if changes_to_apply:
                        __snake_case      :    List[str]  			=getattr(a							,   state.pop('''type'''						)						)
                        __snake_case      :    Union[str, Any]  			=component_class(**a						)
                        setattr(self.backend_tokenizer							,   a							,   a						)


      @property
      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            if self._mask_token is None:
                  if self.verbose:
                        logger.error('''Using mask_token, but it is not set yet.'''						)
                  return None
            return str(self._mask_token						)


      @mask_token.setter
      def _UpperCamelCase						(    self				:	Dict							,   a				:	Tuple						):





            """simple docstring"""







            __snake_case      :    int  			=AddedToken(a							,   lstrip=a							,   rstrip=a						) if isinstance(a							,   a						) else value
            __snake_case      :    Union[str, Any]  			=value


      def _UpperCamelCase						(    self				:	Any							,   *a				:	List[str]							,   **a				:	List[str]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=kwargs.get('''is_split_into_words'''							,   a						)

            if is_split_into_words and not self.add_prefix_space:
                  raise ValueError(
                      f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                      '''to use it with pretokenized inputs.'''						)

            return super()._batch_encode_plus(*a							,   **a						)


      def _UpperCamelCase						(    self				:	Optional[int]							,   *a				:	Any							,   **a				:	Tuple						):





            """simple docstring"""







            __snake_case      :    Any  			=kwargs.get('''is_split_into_words'''							,   a						)

            if is_split_into_words and not self.add_prefix_space:
                  raise ValueError(
                      f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                      '''to use it with pretokenized inputs.'''						)

            return super()._encode_plus(*a							,   **a						)


      def _UpperCamelCase						(    self				:	Tuple							,   a				:	str							,   a				:	Optional[str] = None						):





            """simple docstring"""







            __snake_case      :    Tuple  			=self._tokenizer.model.save(a							,   name=a						)
            return tuple(a						)


      def _UpperCamelCase						(    self				:	Union[str, Any]							,   a				:	Union[str, Any]							,   a				:	Optional[int]=None						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
            if token_ids_a is None:
                  return output

            return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







      def _UpperCamelCase						(    self				:	Union[str, Any]							,   a				:	List[int]							,   a				:	Optional[List[int]] = None						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=[self.sep_token_id]
            __snake_case      :    int  			=[self.cls_token_id]

            if token_ids_a is None:
                  return len(cls + token_ids_a + sep						) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]










"""simple docstring"""





import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
    import torch

    from transformers import (
        MraForMaskedLM,
        MraForMultipleChoice,
        MraForQuestionAnswering,
        MraForSequenceClassification,
        MraForTokenClassification,
        MraModel,
    )
    from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST


class      _lowercase :


      def __init__(    self				:	str							,   a				:	List[str]							,   a				:	Any=2							,   a				:	int=8							,   a				:	List[str]=True							,   a				:	Union[str, Any]=True							,   a				:	Union[str, Any]=True							,   a				:	Optional[int]=True							,   a				:	Union[str, Any]=9_9							,   a				:	Dict=1_6							,   a				:	Union[str, Any]=5							,   a				:	int=2							,   a				:	List[str]=3_6							,   a				:	int="gelu"							,   a				:	Optional[int]=0.0							,   a				:	Any=0.0							,   a				:	Optional[Any]=5_1_2							,   a				:	Tuple=1_6							,   a				:	Dict=2							,   a				:	Union[str, Any]=0.0_2							,   a				:	Dict=3							,   a				:	Union[str, Any]=4							,   a				:	Optional[int]=None							,   ):





            """simple docstring"""







            __snake_case      :    Any  			=parent
            __snake_case      :    int  			=batch_size
            __snake_case      :    Dict  			=seq_length
            __snake_case      :    Any  			=is_training
            __snake_case      :    Optional[int]  			=use_input_mask
            __snake_case      :    List[Any]  			=use_token_type_ids
            __snake_case      :    List[str]  			=use_labels
            __snake_case      :    Optional[Any]  			=vocab_size
            __snake_case      :    Optional[Any]  			=hidden_size
            __snake_case      :    Optional[Any]  			=num_hidden_layers
            __snake_case      :    Any  			=num_attention_heads
            __snake_case      :    Optional[int]  			=intermediate_size
            __snake_case      :    Dict  			=hidden_act
            __snake_case      :    List[str]  			=hidden_dropout_prob
            __snake_case      :    List[Any]  			=attention_probs_dropout_prob
            __snake_case      :    List[str]  			=max_position_embeddings
            __snake_case      :    Tuple  			=type_vocab_size
            __snake_case      :    Tuple  			=type_sequence_label_size
            __snake_case      :    int  			=initializer_range
            __snake_case      :    Any  			=num_labels
            __snake_case      :    List[str]  			=num_choices
            __snake_case      :    Union[str, Any]  			=scope


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=ids_tensor([self.batch_size, self.seq_length]							,   self.vocab_size						)

            __snake_case      :    int  			=None
            if self.use_input_mask:
                  __snake_case      :    Optional[int]  			=random_attention_mask([self.batch_size, self.seq_length]						)

            __snake_case      :    Optional[Any]  			=None
            if self.use_token_type_ids:
                  __snake_case      :    Optional[int]  			=ids_tensor([self.batch_size, self.seq_length]							,   self.type_vocab_size						)

            __snake_case      :    str  			=None
            __snake_case      :    int  			=None
            __snake_case      :    int  			=None
            if self.use_labels:
                  __snake_case      :    int  			=ids_tensor([self.batch_size]							,   self.type_sequence_label_size						)
                  __snake_case      :    str  			=ids_tensor([self.batch_size, self.seq_length]							,   self.num_labels						)
                  __snake_case      :    Optional[Any]  			=ids_tensor([self.batch_size]							,   self.num_choices						)

            __snake_case      :    List[str]  			=self.get_config()

            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            return MraConfig(
                vocab_size=self.vocab_size							,   hidden_size=self.hidden_size							,   num_hidden_layers=self.num_hidden_layers							,   num_attention_heads=self.num_attention_heads							,   intermediate_size=self.intermediate_size							,   hidden_act=self.hidden_act							,   hidden_dropout_prob=self.hidden_dropout_prob							,   attention_probs_dropout_prob=self.attention_probs_dropout_prob							,   max_position_embeddings=self.max_position_embeddings							,   type_vocab_size=self.type_vocab_size							,   is_decoder=a							,   initializer_range=self.initializer_range							,   )


      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            __snake_case      :    int  			=self.get_config()
            __snake_case      :    Optional[Any]  			=3_0_0
            return config


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            (
                (
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				
            )      :    Tuple  			=self.prepare_config_and_inputs()

            __snake_case      :    Dict  			=True
            __snake_case      :    Any  			=floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
            __snake_case      :    Dict  			=ids_tensor([self.batch_size, self.seq_length]							,   vocab_size=2						)

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                encoder_hidden_states,
                encoder_attention_mask,
            )


      def _UpperCamelCase						(    self				:	List[str]							,   a				:	List[Any]							,   a				:	int							,   a				:	Optional[int]							,   a				:	List[Any]							,   a				:	Tuple							,   a				:	Tuple							,   a				:	List[Any]						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=MraModel(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Union[str, Any]  			=model(a							,   attention_mask=a							,   token_type_ids=a						)
            __snake_case      :    Any  			=model(a							,   token_type_ids=a						)
            __snake_case      :    int  			=model(a						)
            self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)						)


      def _UpperCamelCase						(    self				:	str							,   a				:	Tuple							,   a				:	Any							,   a				:	int							,   a				:	Optional[int]							,   a				:	Union[str, Any]							,   a				:	int							,   a				:	Tuple							,   a				:	Optional[Any]							,   a				:	Optional[int]							,   ):





            """simple docstring"""







            __snake_case      :    Any  			=True
            __snake_case      :    Union[str, Any]  			=MraModel(a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Union[str, Any]  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   encoder_hidden_states=a							,   encoder_attention_mask=a							,   )
            __snake_case      :    List[Any]  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   encoder_hidden_states=a							,   )
            __snake_case      :    List[str]  			=model(a							,   attention_mask=a							,   token_type_ids=a						)
            self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)						)


      def _UpperCamelCase						(    self				:	Tuple							,   a				:	Any							,   a				:	List[Any]							,   a				:	int							,   a				:	str							,   a				:	Optional[Any]							,   a				:	List[str]							,   a				:	List[str]						):





            """simple docstring"""







            __snake_case      :    Tuple  			=MraForMaskedLM(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    str  			=model(a							,   attention_mask=a							,   token_type_ids=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.vocab_size)						)


      def _UpperCamelCase						(    self				:	List[str]							,   a				:	Optional[int]							,   a				:	int							,   a				:	List[str]							,   a				:	int							,   a				:	Any							,   a				:	Union[str, Any]							,   a				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=MraForQuestionAnswering(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Dict  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   start_positions=a							,   end_positions=a							,   )
            self.parent.assertEqual(result.start_logits.shape							,   (self.batch_size, self.seq_length)						)
            self.parent.assertEqual(result.end_logits.shape							,   (self.batch_size, self.seq_length)						)


      def _UpperCamelCase						(    self				:	Dict							,   a				:	Dict							,   a				:	Dict							,   a				:	Optional[Any]							,   a				:	int							,   a				:	int							,   a				:	List[str]							,   a				:	Any						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=self.num_labels
            __snake_case      :    Optional[Any]  			=MraForSequenceClassification(a						)
            model.to(a						)
            model.eval()
            __snake_case      :    str  			=model(a							,   attention_mask=a							,   token_type_ids=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.num_labels)						)


      def _UpperCamelCase						(    self				:	Optional[int]							,   a				:	Tuple							,   a				:	Any							,   a				:	Optional[int]							,   a				:	Any							,   a				:	int							,   a				:	str							,   a				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    Tuple  			=self.num_labels
            __snake_case      :    Optional[int]  			=MraForTokenClassification(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    List[str]  			=model(a							,   attention_mask=a							,   token_type_ids=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.num_labels)						)


      def _UpperCamelCase						(    self				:	str							,   a				:	Tuple							,   a				:	Tuple							,   a				:	Optional[Any]							,   a				:	Dict							,   a				:	Dict							,   a				:	Dict							,   a				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.num_choices
            __snake_case      :    Optional[int]  			=MraForMultipleChoice(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    List[str]  			=input_ids.unsqueeze(1						).expand(-1							,   self.num_choices							,   -1						).contiguous()
            __snake_case      :    Tuple  			=token_type_ids.unsqueeze(1						).expand(-1							,   self.num_choices							,   -1						).contiguous()
            __snake_case      :    Any  			=input_mask.unsqueeze(1						).expand(-1							,   self.num_choices							,   -1						).contiguous()
            __snake_case      :    int  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   labels=a							,   )
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.num_choices)						)







      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.prepare_config_and_inputs()
            (
                (
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				
            )      :    Optional[int]  			=config_and_inputs
            __snake_case      :    Tuple  			={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
            return config, inputs_dict



@require_torch
class      _lowercase (     lowerCAmelCase		,  unittest.TestCase						):
      _a   :     Union[str, Any]								=   (
          (
              MraModel,
              MraForMaskedLM,
              MraForMultipleChoice,
              MraForQuestionAnswering,
              MraForSequenceClassification,
              MraForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _a   :     int								=   False
      _a   :     Union[str, Any]								=   False
      _a   :     Dict								=   False
      _a   :     Dict								=   False

      _a   :     Any								=   ()


      def _UpperCamelCase						(    self				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=MraModelTester(self						)
            __snake_case      :    List[Any]  			=ConfigTester(self							,   config_class=a							,   hidden_size=3_7						)


      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            self.config_tester.run_common_tests()


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Dict  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*a						)


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=self.model_tester.prepare_config_and_inputs()
            for type in ["absolute", "relative_key", "relative_key_query"]:
                  __snake_case      :    Any  			=type
                  self.model_tester.create_and_check_model(*a						)


      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*a						)


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_multiple_choice(*a						)


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_question_answering(*a						)


      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            __snake_case      :    str  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_sequence_classification(*a						)


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*a						)


      @slow
      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __snake_case      :    Tuple  			=MraModel.from_pretrained(a						)
                  self.assertIsNotNone(a						)







      @unittest.skip(reason='''MRA does not output attentions'''						)
      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            return



@require_torch
class      _lowercase (     unittest.TestCase						):


      @slow
      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    str  			=MraModel.from_pretrained('''uw-madison/mra-base-512-4'''						)
            __snake_case      :    Optional[int]  			=torch.arange(2_5_6						).unsqueeze(0						)

            with torch.no_grad():
                  __snake_case      :    List[str]  			=model(a						)[0]

            __snake_case      :    Any  			=torch.Size((1, 2_5_6, 7_6_8)						)
            self.assertEqual(output.shape							,   a						)

            __snake_case      :    int  			=torch.tensor(
                [[[-0.0_1_4_0, 0.0_8_3_0, -0.0_3_8_1], [0.1_5_4_6, 0.1_4_0_2, 0.0_2_2_0], [0.1_1_6_2, 0.0_8_5_1, 0.0_1_6_5]]]						)

            self.assertTrue(torch.allclose(output[:, :3, :3]							,   a							,   atol=1e-4						)						)


      @slow
      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            __snake_case      :    Any  			=MraForMaskedLM.from_pretrained('''uw-madison/mra-base-512-4'''						)
            __snake_case      :    Union[str, Any]  			=torch.arange(2_5_6						).unsqueeze(0						)

            with torch.no_grad():
                  __snake_case      :    Optional[int]  			=model(a						)[0]

            __snake_case      :    Union[str, Any]  			=5_0_2_6_5

            __snake_case      :    List[str]  			=torch.Size((1, 2_5_6, vocab_size)						)
            self.assertEqual(output.shape							,   a						)

            __snake_case      :    str  			=torch.tensor(
                [[[9.2_5_9_5, -3.6_0_3_8, 1_1.8_8_1_9], [9.3_8_6_9, -3.2_6_9_3, 1_1.0_9_5_6], [1_1.8_5_2_4, -3.4_9_3_8, 1_3.1_2_1_0]]]						)

            self.assertTrue(torch.allclose(output[:, :3, :3]							,   a							,   atol=1e-4						)						)







      @slow
      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=MraForMaskedLM.from_pretrained('''uw-madison/mra-base-4096-8-d3'''						)
            __snake_case      :    Optional[int]  			=torch.arange(4_0_9_6						).unsqueeze(0						)

            with torch.no_grad():
                  __snake_case      :    Tuple  			=model(a						)[0]

            __snake_case      :    Optional[int]  			=5_0_2_6_5

            __snake_case      :    Tuple  			=torch.Size((1, 4_0_9_6, vocab_size)						)
            self.assertEqual(output.shape							,   a						)

            __snake_case      :    List[str]  			=torch.tensor(
                [[[5.4_7_8_9, -2.3_5_6_4, 7.5_0_6_4], [7.9_0_6_7, -1.3_3_6_9, 9.9_6_6_8], [9.0_7_1_2, -1.8_1_0_6, 7.0_3_8_0]]]						)

            self.assertTrue(torch.allclose(output[:, :3, :3]							,   a							,   atol=1e-4						)						)


