
import inspect
import unittest
import warnings

from transformers import DeiTConfig
from transformers.models.auto import get_values
from transformers.testing_utils import (
    require_accelerate,
    require_torch,
    require_torch_gpu,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch
					from torch import nn

					from transformers import (
					    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
					    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
					    MODEL_MAPPING,
					    DeiTForImageClassification,
					    DeiTForImageClassificationWithTeacher,
					    DeiTForMaskedImageModeling,
					    DeiTModel,
					)
					from transformers.models.deit.modeling_deit import DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
					from PIL import Image

					from transformers import DeiTImageProcessor



class       _snake_case   :


	def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=13					,  SCREAMING_SNAKE_CASE_=30					,  SCREAMING_SNAKE_CASE_=2					,  SCREAMING_SNAKE_CASE_=3					,  SCREAMING_SNAKE_CASE_=True					,  SCREAMING_SNAKE_CASE_=True					,  SCREAMING_SNAKE_CASE_=32					,  SCREAMING_SNAKE_CASE_=5					,  SCREAMING_SNAKE_CASE_=4					,  SCREAMING_SNAKE_CASE_=37					,  SCREAMING_SNAKE_CASE_="gelu"					,  SCREAMING_SNAKE_CASE_=0.1					,  SCREAMING_SNAKE_CASE_=0.1					,  SCREAMING_SNAKE_CASE_=10					,  SCREAMING_SNAKE_CASE_=0.0_2					,  SCREAMING_SNAKE_CASE_=3					,  SCREAMING_SNAKE_CASE_=None					,  SCREAMING_SNAKE_CASE_=2					,  ):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      parent
			lowercase__  :   Optional[int]    		=      batch_size
			lowercase__  :   Any    		=      image_size
			lowercase__  :   Optional[Any]    		=      patch_size
			lowercase__  :   Optional[Any]    		=      num_channels
			lowercase__  :   Tuple    		=      is_training
			lowercase__  :   Optional[Any]    		=      use_labels
			lowercase__  :   Union[str, Any]    		=      hidden_size
			lowercase__  :   Tuple    		=      num_hidden_layers
			lowercase__  :   Union[str, Any]    		=      num_attention_heads
			lowercase__  :   Union[str, Any]    		=      intermediate_size
			lowercase__  :   Any    		=      hidden_act
			lowercase__  :   Tuple    		=      hidden_dropout_prob
			lowercase__  :   Dict    		=      attention_probs_dropout_prob
			lowercase__  :   Any    		=      type_sequence_label_size
			lowercase__  :   Tuple    		=      initializer_range
			lowercase__  :   List[Any]    		=      scope
			lowercase__  :   Optional[int]    		=      encoder_stride

			# in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
			lowercase__  :   List[str]    		=      (image_size // patch_size) ** 2
			lowercase__  :   List[str]    		=      num_patches + 2


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Union[str, Any]    		=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

			lowercase__  :   List[Any]    		=      None
			if self.use_labels:
					lowercase__  :   List[str]    		=      ids_tensor([self.batch_size]					,  self.type_sequence_label_size)

			lowercase__  :   Dict    		=      self.get_config()

			return config, pixel_values, labels


	def      lowercase__     (					self):


			'''simple docstring'''




			return DeiTConfig(
			    image_size=self.image_size					,  patch_size=self.patch_size					,  num_channels=self.num_channels					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  is_decoder=__lowerCamelCase					,  initializer_range=self.initializer_range					,  encoder_stride=self.encoder_stride					,  )


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   Optional[int]    		=      DeiTModel(config=__lowerCamelCase)
			model.to(__lowerCamelCase)
			model.eval()
			lowercase__  :   List[Any]    		=      model(__lowerCamelCase)
			self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size))


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      DeiTForMaskedImageModeling(config=__lowerCamelCase)
			model.to(__lowerCamelCase)
			model.eval()
			lowercase__  :   Optional[int]    		=      model(__lowerCamelCase)
			self.parent.assertEqual(
			    result.reconstruction.shape					,  (self.batch_size, self.num_channels, self.image_size, self.image_size))

			# test greyscale images
			lowercase__  :   List[str]    		=      1
			lowercase__  :   Optional[int]    		=      DeiTForMaskedImageModeling(__lowerCamelCase)
			model.to(__lowerCamelCase)
			model.eval()

			lowercase__  :   Dict    		=      floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
			lowercase__  :   Optional[int]    		=      model(__lowerCamelCase)
			self.parent.assertEqual(result.reconstruction.shape					,  (self.batch_size, 1, self.image_size, self.image_size))


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   str    		=      self.type_sequence_label_size
			lowercase__  :   List[str]    		=      DeiTForImageClassification(__lowerCamelCase)
			model.to(__lowerCamelCase)
			model.eval()
			lowercase__  :   Tuple    		=      model(__lowerCamelCase					,  labels=__lowerCamelCase)
			self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.type_sequence_label_size))

			# test greyscale images
			lowercase__  :   Any    		=      1
			lowercase__  :   str    		=      DeiTForImageClassification(__lowerCamelCase)
			model.to(__lowerCamelCase)
			model.eval()

			lowercase__  :   Tuple    		=      floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
			lowercase__  :   Tuple    		=      model(__lowerCamelCase					,  labels=__lowerCamelCase)
			self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.type_sequence_label_size))



	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Dict    		=      self.prepare_config_and_inputs()
			(
			    lowercase__ 
			)  :   Tuple    		=      config_and_inputs
			lowercase__  :   int    		=      {"pixel_values": pixel_values}
			return config, inputs_dict



@require_torch
class       _snake_case   (	_A  ,						_A  ,						unittest.TestCase							):
	__lowerCAmelCase  :     int        =			(
	    (
	        DeiTModel,
	        DeiTForImageClassification,
	        DeiTForImageClassificationWithTeacher,
	        DeiTForMaskedImageModeling,
	    )
	    if is_torch_available()
	    else ()
	)
	__lowerCAmelCase  :     int        =			(
	    {
	        'feature-extraction': DeiTModel,
	        'image-classification': (DeiTForImageClassification, DeiTForImageClassificationWithTeacher),
	    }
	    if is_torch_available()
	    else {}
	)

	__lowerCAmelCase  :     Optional[int]        =			False
	__lowerCAmelCase  :     Union[str, Any]        =			False
	__lowerCAmelCase  :     Dict        =			False


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   str    		=      DeiTModelTester(self)
			lowercase__  :   Optional[int]    		=      ConfigTester(self					,  config_class=__lowerCamelCase					,  has_text_modality=__lowerCamelCase					,  hidden_size=37)


	def      lowercase__     (					self):


			'''simple docstring'''




			self.config_tester.run_common_tests()


	@unittest.skip(reason="""DeiT does not use inputs_embeds""")
	def      lowercase__     (					self):


			'''simple docstring'''




			pass


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Dict    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Dict    		=      model_class(__lowerCamelCase)
					self.assertIsInstance(model.get_input_embeddings()					,  (nn.Module))
					lowercase__  :   Any    		=      model.get_output_embeddings()
					self.assertTrue(x is None or isinstance(__lowerCamelCase					,  nn.Linear))


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[Any]    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Union[str, Any]    		=      model_class(__lowerCamelCase)
					lowercase__  :   Optional[int]    		=      inspect.signature(model.forward)
					# signature.parameters is an OrderedDict => so arg_names order is deterministic
					lowercase__  :   Any    		=      [*signature.parameters.keys()]

					lowercase__  :   Any    		=      ["pixel_values"]
					self.assertListEqual(arg_names[:1]					,  __lowerCamelCase)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Any    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*__lowerCamelCase)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_image_modeling(*__lowerCamelCase)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_image_classification(*__lowerCamelCase)


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=False):


			'''simple docstring'''




			lowercase__  :   Union[str, Any]    		=      super()._prepare_for_class(__lowerCamelCase					,  __lowerCamelCase					,  return_labels=__lowerCamelCase)

			if return_labels:
					if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
							del inputs_dict["labels"]

			return inputs_dict


	def      lowercase__     (					self):


			'''simple docstring'''




			if not self.model_tester.is_training:
					return

			lowercase__  :   Optional[int]    		=      self.model_tester.prepare_config_and_inputs_for_common()
			lowercase__  :   Dict    		=      True

			for model_class in self.all_model_classes:
					# DeiTForImageClassificationWithTeacher supports inference-only
					if (
					    model_class in get_values(__lowerCamelCase)
					    or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
					):
							continue
					lowercase__  :   Union[str, Any]    		=      model_class(__lowerCamelCase)
					model.to(__lowerCamelCase)
					model.train()
					lowercase__  :   Union[str, Any]    		=      self._prepare_for_class(__lowerCamelCase					,  __lowerCamelCase					,  return_labels=__lowerCamelCase)
					lowercase__  :   Dict    		=      model(**__lowerCamelCase).loss
					loss.backward()


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[Any]    		=      self.model_tester.prepare_config_and_inputs_for_common()
			if not self.model_tester.is_training:
					return

			lowercase__  :   Tuple    		=      False
			lowercase__  :   Any    		=      True

			for model_class in self.all_model_classes:
					if model_class in get_values(__lowerCamelCase) or not model_class.supports_gradient_checkpointing:
							continue
					# DeiTForImageClassificationWithTeacher supports inference-only
					if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
							continue
					lowercase__  :   List[str]    		=      model_class(__lowerCamelCase)
					model.gradient_checkpointing_enable()
					model.to(__lowerCamelCase)
					model.train()
					lowercase__  :   Dict    		=      self._prepare_for_class(__lowerCamelCase					,  __lowerCamelCase					,  return_labels=__lowerCamelCase)
					lowercase__  :   Tuple    		=      model(**__lowerCamelCase).loss
					loss.backward()


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[Any]    		=      self.model_tester.prepare_config_and_inputs_for_common()

			lowercase__  :   int    		=      [
			    {"title": "multi_label_classification", "num_labels": 2, "dtype": torch.float},
			    {"title": "single_label_classification", "num_labels": 1, "dtype": torch.long},
			    {"title": "regression", "num_labels": 1, "dtype": torch.float},
			]

			for model_class in self.all_model_classes:
					if (
					    model_class
					    not in [
					        *get_values(__lowerCamelCase),
					        *get_values(__lowerCamelCase),
					    ]
					    or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
					):
							continue

					for problem_type in problem_types:
							with self.subTest(msg=f'Testing {model_class} with {problem_type["title"]}'):
									lowercase__  :   Tuple    		=      problem_type["title"]
									lowercase__  :   Optional[Any]    		=      problem_type["num_labels"]

									lowercase__  :   List[str]    		=      model_class(__lowerCamelCase)
									model.to(__lowerCamelCase)
									model.train()

									lowercase__  :   List[Any]    		=      self._prepare_for_class(__lowerCamelCase					,  __lowerCamelCase					,  return_labels=__lowerCamelCase)

									if problem_type["num_labels"] > 1:
											lowercase__  :   List[str]    		=      inputs["labels"].unsqueeze(1).repeat(1					,  problem_type["""num_labels"""])

									lowercase__  :   int    		=      inputs["labels"].to(problem_type["""dtype"""])

									# This tests that we do not trigger the warning form PyTorch "Using a target size that is different
									# to the input size. This will likely lead to incorrect results due to broadcasting. Please ensure
									# they have the same size." which is a symptom something in wrong for the regression problem.
									# See https://github.com/huggingface/transformers/issues/11780
									with warnings.catch_warnings(record=__lowerCamelCase) as warning_list:
											lowercase__  :   Optional[Any]    		=      model(**__lowerCamelCase).loss
									for w in warning_list:
											if "Using a target size that is different to the input size" in str(w.message):
													raise ValueError(
													    f'Something is going wrong in the regression problem: intercepted {w.message}')

									loss.backward()



	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			for model_name in DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowercase__  :   Optional[Any]    		=      DeiTModel.from_pretrained(__lowerCamelCase)
					self.assertIsNotNone(__lowerCamelCase)

def 		UpperCamelCase	(      )		->	Tuple:


		'''simple docstring'''






		lowercase__  :   Optional[int]    		=      Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""   )
		return image



@require_torch
@require_vision
class       _snake_case   (	unittest.TestCase							):


	@cached_property
	def      lowercase__     (					self):


			'''simple docstring'''




			return (
			    DeiTImageProcessor.from_pretrained("""facebook/deit-base-distilled-patch16-224""")
			    if is_vision_available()
			    else None
			)


	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[int]    		=      DeiTForImageClassificationWithTeacher.from_pretrained("""facebook/deit-base-distilled-patch16-224""").to(
			    __lowerCamelCase)

			lowercase__  :   List[Any]    		=      self.default_image_processor
			lowercase__  :   List[Any]    		=      prepare_img()
			lowercase__  :   Optional[Any]    		=      image_processor(images=__lowerCamelCase					,  return_tensors="""pt""").to(__lowerCamelCase)

			# forward pass
			with torch.no_grad():
					lowercase__  :   List[str]    		=      model(**__lowerCamelCase)

			# verify the logits
			lowercase__  :   str    		=      torch.Size((1, 10_00))
			self.assertEqual(outputs.logits.shape					,  __lowerCamelCase)

			lowercase__  :   List[str]    		=      torch.tensor([-1.0_2_6_6, 0.1_9_1_2, -1.2_8_6_1]).to(__lowerCamelCase)

			self.assertTrue(torch.allclose(outputs.logits[0, :3]					,  __lowerCamelCase					,  atol=1E-4))



	@slow
	@require_accelerate
	@require_torch_gpu
	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   str    		=      DeiTModel.from_pretrained(
			    """facebook/deit-base-distilled-patch16-224"""					,  torch_dtype=torch.floataa					,  device_map="""auto""")
			lowercase__  :   Dict    		=      self.default_image_processor

			lowercase__  :   Optional[int]    		=      prepare_img()
			lowercase__  :   Union[str, Any]    		=      image_processor(images=__lowerCamelCase					,  return_tensors="""pt""")
			lowercase__  :   Union[str, Any]    		=      inputs.pixel_values.to(__lowerCamelCase)

			# forward pass to make sure inference works in fp16
			with torch.no_grad():
					lowercase__  :   List[str]    		=      model(__lowerCamelCase)








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCamelCase__    :		Dict          =   """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCamelCase__    :		Tuple          =   """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCamelCase__    :		Any          =   """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Optional[Any]:


  '''simple docstring'''






  return float((preds == labels).mean()   )

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_="binary"   )		->	int:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      simple_accuracy(lowercase_ , lowercase_   )
  lowercase__  :   Optional[Any]    		=      float(fa_score(y_true=lowercase_ , y_pred=lowercase_ , average=lowercase_   )   )
  return {
      "accuracy": acc,
      "f1": fa,
  }

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[Any]    		=      {}
  for id_pred, label in zip(lowercase_ , lowercase_   ):
    lowercase__  :   Optional[Any]    		=      F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
    lowercase__  :   Optional[Any]    		=      id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)   )
    else:
      lowercase__  :   Optional[int]    		=      [(pred, label)]
  lowercase__		,    lowercase__  :   List[str]    		=      [], []
  for question, preds_labels in question_map.items():
    lowercase__		,    lowercase__  :   List[str]    		=      zip(*lowercase_   )
    lowercase__  :   Optional[Any]    		=      fa_score(y_true=lowercase_ , y_pred=lowercase_ , average="""macro"""   )
    fas.append(lowercase_   )
    lowercase__  :   List[Any]    		=      int(sum(pred == label for pred, label in preds_labels   ) == len(lowercase_   )   )
    ems.append(lowercase_   )
  lowercase__  :   str    		=      float(sum(lowercase_   ) / len(lowercase_   )   )
  lowercase__  :   str    		=      sum(lowercase_   ) / len(lowercase_   )
  lowercase__  :   List[str]    		=      float(fa_score(y_true=lowercase_ , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION							)
class       _snake_case   (	datasets.Metric							):


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name not in [
       "boolq",
       "cb",
       "copa",
       "multirc",
       "record",
       "rte",
       "wic",
       "wsc",
       "wsc.fixed",
       "axb",
       "axg",
   ]:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
   return datasets.MetricInfo(
       description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(self._get_feature_types())					,  codebase_urls=[]					,  reference_urls=[]					,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name == "record":
     return {
         "predictions": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "prediction_text": datasets.Value("""string"""),
         },
         "references": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "answers": datasets.Sequence(datasets.Value("""string""")),
         },
     }
   elif self.config_name == "multirc":
     return {
         "predictions": {
             "idx": {
                 "answer": datasets.Value("""int64"""),
                 "paragraph": datasets.Value("""int64"""),
                 "question": datasets.Value("""int64"""),
             },
             "prediction": datasets.Value("""int64"""),
         },
         "references": datasets.Value("""int64"""),
     }
   else:
     return {
         "predictions": datasets.Value("""int64"""),
         "references": datasets.Value("""int64"""),
     }



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.config_name == "axb":
     return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   elif self.config_name == "cb":
     return acc_and_fa(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  fa_avg="""macro""")
   elif self.config_name == "record":
     lowercase__  :   Optional[Any]    		=      [
         {
             """qas""": [
                 {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                 for ref in references
             ]
         }
     ]
     lowercase__  :   str    		=      {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
     return evaluate_record(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)[0]
   elif self.config_name == "multirc":
     return evaluate_multirc(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
     return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   else:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






