






"""simple docstring"""





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class      _lowercase (     lowerCAmelCase						):


      def __init__(    self				:	Optional[Any]							,   a				:	NestedDataStructureLike[PathLike]							,   a				:	Optional[NamedSplit] = None							,   a				:	Optional[Features] = None							,   a				:	str = None							,   a				:	bool = False							,   a				:	bool = False							,   a				:	Optional[str] = None							,   a				:	Optional[int] = None							,   **a				:	str							,   ):





            """simple docstring"""







            super().__init__(
                a							,   split=a							,   features=a							,   cache_dir=a							,   keep_in_memory=a							,   streaming=a							,   num_proc=a							,   **a							,   )
            __snake_case      :    Any  			=field
            __snake_case      :    Optional[int]  			=path_or_paths if isinstance(a							,   a						) else {self.split: path_or_paths}
            __snake_case      :    int  			=Json(
                cache_dir=a							,   data_files=a							,   features=a							,   field=a							,   **a							,   )







      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            if self.streaming:
                  __snake_case      :    List[str]  			=self.builder.as_streaming_dataset(split=self.split						)
            # Build regular (map-style) dataset
            else:
                  __snake_case      :    Any  			=None
                  __snake_case      :    Union[str, Any]  			=None
                  __snake_case      :    List[Any]  			=None
                  __snake_case      :    List[Any]  			=None

                  self.builder.download_and_prepare(
                      download_config=a							,   download_mode=a							,   verification_mode=a							,   base_path=a							,   num_proc=self.num_proc							,   )
                  __snake_case      :    Optional[int]  			=self.builder.as_dataset(
                      split=self.split							,   verification_mode=a							,   in_memory=self.keep_in_memory						)
            return dataset



class      _lowercase :


      def __init__(    self				:	Any							,   a				:	Dataset							,   a				:	Union[PathLike, BinaryIO]							,   a				:	Optional[int] = None							,   a				:	Optional[int] = None							,   **a				:	Optional[Any]							,   ):





            """simple docstring"""







            if num_proc is not None and num_proc <= 0:
                  raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''						)

            __snake_case      :    List[str]  			=dataset
            __snake_case      :    Dict  			=path_or_buf
            __snake_case      :    Optional[Any]  			=batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
            __snake_case      :    List[str]  			=num_proc
            __snake_case      :    Union[str, Any]  			='''utf-8'''
            __snake_case      :    List[str]  			=to_json_kwargs


      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.to_json_kwargs.pop('''path_or_buf'''							,   a						)
            __snake_case      :    Tuple  			=self.to_json_kwargs.pop('''orient'''							,   '''records'''						)
            __snake_case      :    Optional[int]  			=self.to_json_kwargs.pop('''lines'''							,   True if orient == '''records''' else False						)
            __snake_case      :    str  			=self.to_json_kwargs.pop('''index'''							,   False if orient in ['''split''', '''table'''] else True						)
            __snake_case      :    Any  			=self.to_json_kwargs.pop('''compression'''							,   a						)

            if compression not in [None, "infer", "gzip", "bz2", "xz"]:
                  raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''						)

            if isinstance(self.path_or_buf							,   (str, bytes, os.PathLike)						):
                  with fsspec.open(self.path_or_buf							,   '''wb'''							,   compression=a						) as buffer:
                        __snake_case      :    Optional[int]  			=self._write(file_obj=a							,   orient=a							,   lines=a							,   index=a							,   **self.to_json_kwargs						)
            else:
                  if compression:
                        raise NotImplementedError(
                            f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
                            ''' was passed. Please provide a local path instead.'''						)
                  __snake_case      :    List[str]  			=self._write(
                      file_obj=self.path_or_buf							,   orient=a							,   lines=a							,   index=a							,   **self.to_json_kwargs						)
            return written


      def _UpperCamelCase						(    self				:	str							,   a				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case			,				__snake_case			,				__snake_case			,				__snake_case			,				__snake_case      :    List[str]  			=args

            __snake_case      :    List[str]  			=query_table(
                table=self.dataset.data							,   key=slice(a							,   offset + self.batch_size						)							,   indices=self.dataset._indices							,   )
            __snake_case      :    Dict  			=batch.to_pandas().to_json(
                path_or_buf=a							,   orient=a							,   lines=a							,   index=a							,   **a						)
            if not json_str.endswith('''\n'''						):
                  json_str += "\n"
            return json_str.encode(self.encoding						)







      def _UpperCamelCase						(    self				:	Optional[int]							,   a				:	BinaryIO							,   a				:	int							,   a				:	Dict							,   a				:	Tuple							,   **a				:	Any							,   ):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=0

            if self.num_proc is None or self.num_proc == 1:
                  for offset in logging.tqdm(
                      range(0							,   len(self.dataset						)							,   self.batch_size						)							,   unit='''ba'''							,   disable=not logging.is_progress_bar_enabled()							,   desc='''Creating json from Arrow format'''							,   ):
                        __snake_case      :    Tuple  			=self._batch_json((offset, orient, lines, index, to_json_kwargs)						)
                        written += file_obj.write(a						)
            else:
                  __snake_case			,				__snake_case      :    Optional[Any]  			=len(self.dataset						), self.batch_size
                  with multiprocessing.Pool(self.num_proc						) as pool:
                        for json_str in logging.tqdm(
                            pool.imap(
                                self._batch_json							,   [(offset, orient, lines, index, to_json_kwargs) for offset in range(0							,   a							,   a						)]							,   )							,   total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size							,   unit='''ba'''							,   disable=not logging.is_progress_bar_enabled()							,   desc='''Creating json from Arrow format'''							,   ):
                              written += file_obj.write(a						)

            return written










"""simple docstring"""





import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
    import torch

    from transformers import (
        MraForMaskedLM,
        MraForMultipleChoice,
        MraForQuestionAnswering,
        MraForSequenceClassification,
        MraForTokenClassification,
        MraModel,
    )
    from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST


class      _lowercase :


      def __init__(    self				:	str							,   a				:	List[str]							,   a				:	Any=2							,   a				:	int=8							,   a				:	List[str]=True							,   a				:	Union[str, Any]=True							,   a				:	Union[str, Any]=True							,   a				:	Optional[int]=True							,   a				:	Union[str, Any]=9_9							,   a				:	Dict=1_6							,   a				:	Union[str, Any]=5							,   a				:	int=2							,   a				:	List[str]=3_6							,   a				:	int="gelu"							,   a				:	Optional[int]=0.0							,   a				:	Any=0.0							,   a				:	Optional[Any]=5_1_2							,   a				:	Tuple=1_6							,   a				:	Dict=2							,   a				:	Union[str, Any]=0.0_2							,   a				:	Dict=3							,   a				:	Union[str, Any]=4							,   a				:	Optional[int]=None							,   ):





            """simple docstring"""







            __snake_case      :    Any  			=parent
            __snake_case      :    int  			=batch_size
            __snake_case      :    Dict  			=seq_length
            __snake_case      :    Any  			=is_training
            __snake_case      :    Optional[int]  			=use_input_mask
            __snake_case      :    List[Any]  			=use_token_type_ids
            __snake_case      :    List[str]  			=use_labels
            __snake_case      :    Optional[Any]  			=vocab_size
            __snake_case      :    Optional[Any]  			=hidden_size
            __snake_case      :    Optional[Any]  			=num_hidden_layers
            __snake_case      :    Any  			=num_attention_heads
            __snake_case      :    Optional[int]  			=intermediate_size
            __snake_case      :    Dict  			=hidden_act
            __snake_case      :    List[str]  			=hidden_dropout_prob
            __snake_case      :    List[Any]  			=attention_probs_dropout_prob
            __snake_case      :    List[str]  			=max_position_embeddings
            __snake_case      :    Tuple  			=type_vocab_size
            __snake_case      :    Tuple  			=type_sequence_label_size
            __snake_case      :    int  			=initializer_range
            __snake_case      :    Any  			=num_labels
            __snake_case      :    List[str]  			=num_choices
            __snake_case      :    Union[str, Any]  			=scope


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=ids_tensor([self.batch_size, self.seq_length]							,   self.vocab_size						)

            __snake_case      :    int  			=None
            if self.use_input_mask:
                  __snake_case      :    Optional[int]  			=random_attention_mask([self.batch_size, self.seq_length]						)

            __snake_case      :    Optional[Any]  			=None
            if self.use_token_type_ids:
                  __snake_case      :    Optional[int]  			=ids_tensor([self.batch_size, self.seq_length]							,   self.type_vocab_size						)

            __snake_case      :    str  			=None
            __snake_case      :    int  			=None
            __snake_case      :    int  			=None
            if self.use_labels:
                  __snake_case      :    int  			=ids_tensor([self.batch_size]							,   self.type_sequence_label_size						)
                  __snake_case      :    str  			=ids_tensor([self.batch_size, self.seq_length]							,   self.num_labels						)
                  __snake_case      :    Optional[Any]  			=ids_tensor([self.batch_size]							,   self.num_choices						)

            __snake_case      :    List[str]  			=self.get_config()

            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            return MraConfig(
                vocab_size=self.vocab_size							,   hidden_size=self.hidden_size							,   num_hidden_layers=self.num_hidden_layers							,   num_attention_heads=self.num_attention_heads							,   intermediate_size=self.intermediate_size							,   hidden_act=self.hidden_act							,   hidden_dropout_prob=self.hidden_dropout_prob							,   attention_probs_dropout_prob=self.attention_probs_dropout_prob							,   max_position_embeddings=self.max_position_embeddings							,   type_vocab_size=self.type_vocab_size							,   is_decoder=a							,   initializer_range=self.initializer_range							,   )


      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            __snake_case      :    int  			=self.get_config()
            __snake_case      :    Optional[Any]  			=3_0_0
            return config


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            (
                (
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				
            )      :    Tuple  			=self.prepare_config_and_inputs()

            __snake_case      :    Dict  			=True
            __snake_case      :    Any  			=floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
            __snake_case      :    Dict  			=ids_tensor([self.batch_size, self.seq_length]							,   vocab_size=2						)

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                encoder_hidden_states,
                encoder_attention_mask,
            )


      def _UpperCamelCase						(    self				:	List[str]							,   a				:	List[Any]							,   a				:	int							,   a				:	Optional[int]							,   a				:	List[Any]							,   a				:	Tuple							,   a				:	Tuple							,   a				:	List[Any]						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=MraModel(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Union[str, Any]  			=model(a							,   attention_mask=a							,   token_type_ids=a						)
            __snake_case      :    Any  			=model(a							,   token_type_ids=a						)
            __snake_case      :    int  			=model(a						)
            self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)						)


      def _UpperCamelCase						(    self				:	str							,   a				:	Tuple							,   a				:	Any							,   a				:	int							,   a				:	Optional[int]							,   a				:	Union[str, Any]							,   a				:	int							,   a				:	Tuple							,   a				:	Optional[Any]							,   a				:	Optional[int]							,   ):





            """simple docstring"""







            __snake_case      :    Any  			=True
            __snake_case      :    Union[str, Any]  			=MraModel(a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Union[str, Any]  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   encoder_hidden_states=a							,   encoder_attention_mask=a							,   )
            __snake_case      :    List[Any]  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   encoder_hidden_states=a							,   )
            __snake_case      :    List[str]  			=model(a							,   attention_mask=a							,   token_type_ids=a						)
            self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)						)


      def _UpperCamelCase						(    self				:	Tuple							,   a				:	Any							,   a				:	List[Any]							,   a				:	int							,   a				:	str							,   a				:	Optional[Any]							,   a				:	List[str]							,   a				:	List[str]						):





            """simple docstring"""







            __snake_case      :    Tuple  			=MraForMaskedLM(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    str  			=model(a							,   attention_mask=a							,   token_type_ids=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.vocab_size)						)


      def _UpperCamelCase						(    self				:	List[str]							,   a				:	Optional[int]							,   a				:	int							,   a				:	List[str]							,   a				:	int							,   a				:	Any							,   a				:	Union[str, Any]							,   a				:	Union[str, Any]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=MraForQuestionAnswering(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    Dict  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   start_positions=a							,   end_positions=a							,   )
            self.parent.assertEqual(result.start_logits.shape							,   (self.batch_size, self.seq_length)						)
            self.parent.assertEqual(result.end_logits.shape							,   (self.batch_size, self.seq_length)						)


      def _UpperCamelCase						(    self				:	Dict							,   a				:	Dict							,   a				:	Dict							,   a				:	Optional[Any]							,   a				:	int							,   a				:	int							,   a				:	List[str]							,   a				:	Any						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=self.num_labels
            __snake_case      :    Optional[Any]  			=MraForSequenceClassification(a						)
            model.to(a						)
            model.eval()
            __snake_case      :    str  			=model(a							,   attention_mask=a							,   token_type_ids=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.num_labels)						)


      def _UpperCamelCase						(    self				:	Optional[int]							,   a				:	Tuple							,   a				:	Any							,   a				:	Optional[int]							,   a				:	Any							,   a				:	int							,   a				:	str							,   a				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    Tuple  			=self.num_labels
            __snake_case      :    Optional[int]  			=MraForTokenClassification(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    List[str]  			=model(a							,   attention_mask=a							,   token_type_ids=a							,   labels=a						)
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.num_labels)						)


      def _UpperCamelCase						(    self				:	str							,   a				:	Tuple							,   a				:	Tuple							,   a				:	Optional[Any]							,   a				:	Dict							,   a				:	Dict							,   a				:	Dict							,   a				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.num_choices
            __snake_case      :    Optional[int]  			=MraForMultipleChoice(config=a						)
            model.to(a						)
            model.eval()
            __snake_case      :    List[str]  			=input_ids.unsqueeze(1						).expand(-1							,   self.num_choices							,   -1						).contiguous()
            __snake_case      :    Tuple  			=token_type_ids.unsqueeze(1						).expand(-1							,   self.num_choices							,   -1						).contiguous()
            __snake_case      :    Any  			=input_mask.unsqueeze(1						).expand(-1							,   self.num_choices							,   -1						).contiguous()
            __snake_case      :    int  			=model(
                a							,   attention_mask=a							,   token_type_ids=a							,   labels=a							,   )
            self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.num_choices)						)







      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.prepare_config_and_inputs()
            (
                (
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				(
                __snake_case
            )			,				
            )      :    Optional[int]  			=config_and_inputs
            __snake_case      :    Tuple  			={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
            return config, inputs_dict



@require_torch
class      _lowercase (     lowerCAmelCase		,  unittest.TestCase						):
      _a   :     Union[str, Any]								=   (
          (
              MraModel,
              MraForMaskedLM,
              MraForMultipleChoice,
              MraForQuestionAnswering,
              MraForSequenceClassification,
              MraForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _a   :     int								=   False
      _a   :     Union[str, Any]								=   False
      _a   :     Dict								=   False
      _a   :     Dict								=   False

      _a   :     Any								=   ()


      def _UpperCamelCase						(    self				:	Optional[int]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=MraModelTester(self						)
            __snake_case      :    List[Any]  			=ConfigTester(self							,   config_class=a							,   hidden_size=3_7						)


      def _UpperCamelCase						(    self				:	List[Any]						):





            """simple docstring"""







            self.config_tester.run_common_tests()


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Dict  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*a						)


      def _UpperCamelCase						(    self				:	Any						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=self.model_tester.prepare_config_and_inputs()
            for type in ["absolute", "relative_key", "relative_key_query"]:
                  __snake_case      :    Any  			=type
                  self.model_tester.create_and_check_model(*a						)


      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            __snake_case      :    Optional[Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*a						)


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_multiple_choice(*a						)


      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_question_answering(*a						)


      def _UpperCamelCase						(    self				:	int						):





            """simple docstring"""







            __snake_case      :    str  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_sequence_classification(*a						)


      def _UpperCamelCase						(    self				:	str						):





            """simple docstring"""







            __snake_case      :    Union[str, Any]  			=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*a						)


      @slow
      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __snake_case      :    Tuple  			=MraModel.from_pretrained(a						)
                  self.assertIsNotNone(a						)







      @unittest.skip(reason='''MRA does not output attentions'''						)
      def _UpperCamelCase						(    self				:	Optional[Any]						):





            """simple docstring"""







            return



@require_torch
class      _lowercase (     unittest.TestCase						):


      @slow
      def _UpperCamelCase						(    self				:	Dict						):





            """simple docstring"""







            __snake_case      :    str  			=MraModel.from_pretrained('''uw-madison/mra-base-512-4'''						)
            __snake_case      :    Optional[int]  			=torch.arange(2_5_6						).unsqueeze(0						)

            with torch.no_grad():
                  __snake_case      :    List[str]  			=model(a						)[0]

            __snake_case      :    Any  			=torch.Size((1, 2_5_6, 7_6_8)						)
            self.assertEqual(output.shape							,   a						)

            __snake_case      :    int  			=torch.tensor(
                [[[-0.0_1_4_0, 0.0_8_3_0, -0.0_3_8_1], [0.1_5_4_6, 0.1_4_0_2, 0.0_2_2_0], [0.1_1_6_2, 0.0_8_5_1, 0.0_1_6_5]]]						)

            self.assertTrue(torch.allclose(output[:, :3, :3]							,   a							,   atol=1e-4						)						)


      @slow
      def _UpperCamelCase						(    self				:	List[str]						):





            """simple docstring"""







            __snake_case      :    Any  			=MraForMaskedLM.from_pretrained('''uw-madison/mra-base-512-4'''						)
            __snake_case      :    Union[str, Any]  			=torch.arange(2_5_6						).unsqueeze(0						)

            with torch.no_grad():
                  __snake_case      :    Optional[int]  			=model(a						)[0]

            __snake_case      :    Union[str, Any]  			=5_0_2_6_5

            __snake_case      :    List[str]  			=torch.Size((1, 2_5_6, vocab_size)						)
            self.assertEqual(output.shape							,   a						)

            __snake_case      :    str  			=torch.tensor(
                [[[9.2_5_9_5, -3.6_0_3_8, 1_1.8_8_1_9], [9.3_8_6_9, -3.2_6_9_3, 1_1.0_9_5_6], [1_1.8_5_2_4, -3.4_9_3_8, 1_3.1_2_1_0]]]						)

            self.assertTrue(torch.allclose(output[:, :3, :3]							,   a							,   atol=1e-4						)						)







      @slow
      def _UpperCamelCase						(    self				:	Tuple						):





            """simple docstring"""







            __snake_case      :    List[Any]  			=MraForMaskedLM.from_pretrained('''uw-madison/mra-base-4096-8-d3'''						)
            __snake_case      :    Optional[int]  			=torch.arange(4_0_9_6						).unsqueeze(0						)

            with torch.no_grad():
                  __snake_case      :    Tuple  			=model(a						)[0]

            __snake_case      :    Optional[int]  			=5_0_2_6_5

            __snake_case      :    Tuple  			=torch.Size((1, 4_0_9_6, vocab_size)						)
            self.assertEqual(output.shape							,   a						)

            __snake_case      :    List[str]  			=torch.tensor(
                [[[5.4_7_8_9, -2.3_5_6_4, 7.5_0_6_4], [7.9_0_6_7, -1.3_3_6_9, 9.9_6_6_8], [9.0_7_1_2, -1.8_1_0_6, 7.0_3_8_0]]]						)

            self.assertTrue(torch.allclose(output[:, :3, :3]							,   a							,   atol=1e-4						)						)


