
import inspect
import unittest

from transformers import YolosConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch
					from torch import nn

					from transformers import YolosForObjectDetection, YolosModel
					from transformers.models.yolos.modeling_yolos import YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
					from PIL import Image

					from transformers import AutoImageProcessor



class       _snake_case   :


	def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=13					,  SCREAMING_SNAKE_CASE_=[30, 30]					,  SCREAMING_SNAKE_CASE_=2					,  SCREAMING_SNAKE_CASE_=3					,  SCREAMING_SNAKE_CASE_=True					,  SCREAMING_SNAKE_CASE_=True					,  SCREAMING_SNAKE_CASE_=32					,  SCREAMING_SNAKE_CASE_=5					,  SCREAMING_SNAKE_CASE_=4					,  SCREAMING_SNAKE_CASE_=37					,  SCREAMING_SNAKE_CASE_="gelu"					,  SCREAMING_SNAKE_CASE_=0.1					,  SCREAMING_SNAKE_CASE_=0.1					,  SCREAMING_SNAKE_CASE_=10					,  SCREAMING_SNAKE_CASE_=0.0_2					,  SCREAMING_SNAKE_CASE_=3					,  SCREAMING_SNAKE_CASE_=None					,  SCREAMING_SNAKE_CASE_=8					,  SCREAMING_SNAKE_CASE_=10					,  ):


			'''simple docstring'''




			lowercase__  :   Union[str, Any]    		=      parent
			lowercase__  :   Tuple    		=      batch_size
			lowercase__  :   Optional[int]    		=      image_size
			lowercase__  :   Any    		=      patch_size
			lowercase__  :   Optional[Any]    		=      num_channels
			lowercase__  :   Any    		=      is_training
			lowercase__  :   Dict    		=      use_labels
			lowercase__  :   Tuple    		=      hidden_size
			lowercase__  :   Optional[int]    		=      num_hidden_layers
			lowercase__  :   Optional[int]    		=      num_attention_heads
			lowercase__  :   Optional[int]    		=      intermediate_size
			lowercase__  :   Tuple    		=      hidden_act
			lowercase__  :   List[Any]    		=      hidden_dropout_prob
			lowercase__  :   List[Any]    		=      attention_probs_dropout_prob
			lowercase__  :   Union[str, Any]    		=      type_sequence_label_size
			lowercase__  :   List[Any]    		=      initializer_range
			lowercase__  :   List[str]    		=      num_labels
			lowercase__  :   int    		=      scope
			lowercase__  :   str    		=      n_targets
			lowercase__  :   str    		=      num_detection_tokens
			# we set the expected sequence length (which is used in several tests)
			# expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + num_detection_tokens
			lowercase__  :   List[str]    		=      (image_size[1] // patch_size) * (image_size[0] // patch_size)
			lowercase__  :   List[Any]    		=      num_patches + 1 + self.num_detection_tokens


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[Any]    		=      floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]])

			lowercase__  :   Tuple    		=      None
			if self.use_labels:
					# labels is a list of Dict (each Dict being the labels for a given example in the batch)
					lowercase__  :   Union[str, Any]    		=      []
					for i in range(self.batch_size):
							lowercase__  :   Dict    		=      {}
							lowercase__  :   Dict    		=      torch.randint(
							    high=self.num_labels					,  size=(self.n_targets,)					,  device=__a)
							lowercase__  :   Any    		=      torch.rand(self.n_targets					,  4					,  device=__a)
							labels.append(__a)

			lowercase__  :   Union[str, Any]    		=      self.get_config()

			return config, pixel_values, labels


	def      lowercase__     (					self):


			'''simple docstring'''




			return YolosConfig(
			    image_size=self.image_size					,  patch_size=self.patch_size					,  num_channels=self.num_channels					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  is_decoder=__a					,  initializer_range=self.initializer_range					,  num_detection_tokens=self.num_detection_tokens					,  num_labels=self.num_labels					,  )


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   List[Any]    		=      YolosModel(config=__a)
			model.to(__a)
			model.eval()
			lowercase__  :   Dict    		=      model(__a)
			self.parent.assertEqual(
			    result.last_hidden_state.shape					,  (self.batch_size, self.expected_seq_len, self.hidden_size))


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			lowercase__  :   Any    		=      YolosForObjectDetection(__a)
			model.to(__a)
			model.eval()

			lowercase__  :   Union[str, Any]    		=      model(pixel_values=__a)
			lowercase__  :   Dict    		=      model(__a)

			self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_detection_tokens, self.num_labels + 1))
			self.parent.assertEqual(result.pred_boxes.shape					,  (self.batch_size, self.num_detection_tokens, 4))

			lowercase__  :   List[Any]    		=      model(pixel_values=__a					,  labels=__a)

			self.parent.assertEqual(result.loss.shape					,  ())
			self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_detection_tokens, self.num_labels + 1))
			self.parent.assertEqual(result.pred_boxes.shape					,  (self.batch_size, self.num_detection_tokens, 4))



	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Tuple    		=      self.prepare_config_and_inputs()
			lowercase__  :   Dict    		=      config_and_inputs
			lowercase__  :   Any    		=      {'pixel_values': pixel_values}
			return config, inputs_dict



@require_torch
class       _snake_case   (	UpperCamelCase_  ,						UpperCamelCase_  ,						unittest.TestCase							):
	__lowerCAmelCase  :     Optional[Any]        =			(YolosModel, YolosForObjectDetection) if is_torch_available() else ()
	__lowerCAmelCase  :     Optional[int]        =			(
	    {'feature-extraction': YolosModel, 'object-detection': YolosForObjectDetection} if is_torch_available() else {}
	)

	__lowerCAmelCase  :     str        =			False
	__lowerCAmelCase  :     Optional[int]        =			False
	__lowerCAmelCase  :     Optional[Any]        =			False
	__lowerCAmelCase  :     Optional[int]        =			False


	def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=False):


			'''simple docstring'''




			lowercase__  :   Dict    		=      super()._prepare_for_class(__a					,  __a					,  return_labels=__a)

			if return_labels:
					if model_class.__name__ == "YolosForObjectDetection":
							lowercase__  :   List[str]    		=      []
							for i in range(self.model_tester.batch_size):
									lowercase__  :   str    		=      {}
									lowercase__  :   List[Any]    		=      torch.ones(
									    size=(self.model_tester.n_targets,)					,  device=__a					,  dtype=torch.long)
									lowercase__  :   Dict    		=      torch.ones(
									    self.model_tester.n_targets					,  4					,  device=__a					,  dtype=torch.float)
									labels.append(__a)
							lowercase__  :   str    		=      labels

			return inputs_dict


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Tuple    		=      YolosModelTester(self)
			lowercase__  :   List[str]    		=      ConfigTester(self					,  config_class=__a					,  has_text_modality=__a					,  hidden_size=37)


	def      lowercase__     (					self):


			'''simple docstring'''




			self.config_tester.run_common_tests()


	def      lowercase__     (					self):


			'''simple docstring'''




			pass


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Optional[int]    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Tuple    		=      model_class(__a)
					self.assertIsInstance(model.get_input_embeddings()					,  (nn.Module))
					lowercase__  :   List[Any]    		=      model.get_output_embeddings()
					self.assertTrue(x is None or isinstance(__a					,  nn.Linear))


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Any    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Optional[int]    		=      model_class(__a)
					lowercase__  :   str    		=      inspect.signature(model.forward)
					# signature.parameters is an OrderedDict => so arg_names order is deterministic
					lowercase__  :   Tuple    		=      [*signature.parameters.keys()]

					lowercase__  :   Optional[Any]    		=      ['pixel_values']
					self.assertListEqual(arg_names[:1]					,  __a)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Tuple    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*__a)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Tuple    		=      self.model_tester.prepare_config_and_inputs_for_common()
			lowercase__  :   str    		=      True

			# in YOLOS, the seq_len is different
			lowercase__  :   Any    		=      self.model_tester.expected_seq_len
			for model_class in self.all_model_classes:
					lowercase__  :   List[Any]    		=      True
					lowercase__  :   List[str]    		=      False
					lowercase__  :   List[Any]    		=      True
					lowercase__  :   Any    		=      model_class(__a)
					model.to(__a)
					model.eval()
					with torch.no_grad():
							lowercase__  :   str    		=      model(**self._prepare_for_class(__a					,  __a))
					lowercase__  :   Tuple    		=      outputs.attentions
					self.assertEqual(len(__a)					,  self.model_tester.num_hidden_layers)

					# check that output_attentions also work using config
					del inputs_dict["output_attentions"]
					lowercase__  :   int    		=      True
					lowercase__  :   int    		=      model_class(__a)
					model.to(__a)
					model.eval()
					with torch.no_grad():
							lowercase__  :   List[Any]    		=      model(**self._prepare_for_class(__a					,  __a))
					lowercase__  :   int    		=      outputs.attentions
					self.assertEqual(len(__a)					,  self.model_tester.num_hidden_layers)

					self.assertListEqual(
					    list(attentions[0].shape[-3:])					,  [self.model_tester.num_attention_heads, seq_len, seq_len]					,  )
					lowercase__  :   Optional[int]    		=      len(__a)

					# Check attention is always last and order is fine
					lowercase__  :   List[str]    		=      True
					lowercase__  :   Union[str, Any]    		=      True
					lowercase__  :   List[str]    		=      model_class(__a)
					model.to(__a)
					model.eval()
					with torch.no_grad():
							lowercase__  :   Tuple    		=      model(**self._prepare_for_class(__a					,  __a))

					lowercase__  :   Optional[int]    		=      1
					self.assertEqual(out_len + added_hidden_states					,  len(__a))

					lowercase__  :   Optional[int]    		=      outputs.attentions

					self.assertEqual(len(__a)					,  self.model_tester.num_hidden_layers)
					self.assertListEqual(
					    list(self_attentions[0].shape[-3:])					,  [self.model_tester.num_attention_heads, seq_len, seq_len]					,  )


	def      lowercase__     (					self):


			'''simple docstring'''




			def check_hidden_states_output(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):
					lowercase__  :   Optional[int]    		=      model_class(__a)
					model.to(__a)
					model.eval()

					with torch.no_grad():
							lowercase__  :   List[Any]    		=      model(**self._prepare_for_class(__a					,  __a))

					lowercase__  :   Optional[int]    		=      outputs.hidden_states

					lowercase__  :   List[str]    		=      getattr(
					    self.model_tester					,  """expected_num_hidden_layers"""					,  self.model_tester.num_hidden_layers + 1)
					self.assertEqual(len(__a)					,  __a)

					# YOLOS has a different seq_length
					lowercase__  :   Optional[Any]    		=      self.model_tester.expected_seq_len

					self.assertListEqual(
					    list(hidden_states[0].shape[-2:])					,  [seq_length, self.model_tester.hidden_size]					,  )

			lowercase__  :   List[Any]    		=      self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowercase__  :   Tuple    		=      True
					check_hidden_states_output(__a					,  __a					,  __a)

					# check that output_hidden_states also work using config
					del inputs_dict["output_hidden_states"]
					lowercase__  :   Union[str, Any]    		=      True

					check_hidden_states_output(__a					,  __a					,  __a)


	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   Dict    		=      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_object_detection(*__a)



	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			for model_name in YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowercase__  :   List[str]    		=      YolosModel.from_pretrained(__a)
					self.assertIsNotNone(__a)

def 		UpperCamelCase	(      )		->	int:


		'''simple docstring'''






		lowercase__  :   Dict    		=      Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""   )
		return image



@require_torch
@require_vision
class       _snake_case   (	unittest.TestCase							):


	@cached_property
	def      lowercase__     (					self):


			'''simple docstring'''




			return AutoImageProcessor.from_pretrained("""hustvl/yolos-small""") if is_vision_available() else None



	@slow
	def      lowercase__     (					self):


			'''simple docstring'''




			lowercase__  :   int    		=      YolosForObjectDetection.from_pretrained("""hustvl/yolos-small""").to(__a)

			lowercase__  :   Optional[Any]    		=      self.default_image_processor
			lowercase__  :   List[Any]    		=      prepare_img()
			lowercase__  :   Optional[Any]    		=      image_processor(images=__a					,  return_tensors="""pt""").to(__a)

			# forward pass
			with torch.no_grad():
					lowercase__  :   int    		=      model(inputs.pixel_values)

			# verify outputs
			lowercase__  :   Optional[Any]    		=      torch.Size((1, 1_00, 92))
			self.assertEqual(outputs.logits.shape					,  __a)

			lowercase__  :   Optional[int]    		=      torch.tensor(
			    [[-2_4.0_2_4_8, -1_0.3_0_2_4, -1_4.8_2_9_0], [-4_2.0_3_9_2, -1_6.8_2_0_0, -2_7.4_3_3_4], [-2_7.2_7_4_3, -1_1.8_1_5_4, -1_8.7_1_4_8]]					,  device=__a					,  )
			lowercase__  :   int    		=      torch.tensor(
			    [[0.2_5_5_9, 0.5_4_5_5, 0.4_7_0_6], [0.2_9_8_9, 0.7_2_7_9, 0.1_8_7_5], [0.7_7_3_2, 0.4_0_1_7, 0.4_4_6_2]]					,  device=__a)
			self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]					,  __a					,  atol=1E-4))
			self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3]					,  __a					,  atol=1E-4))

			# verify postprocessing
			lowercase__  :   Optional[int]    		=      image_processor.post_process_object_detection(
			    __a					,  threshold=0.3					,  target_sizes=[image.size[::-1]])[0]
			lowercase__  :   List[str]    		=      torch.tensor([0.9_9_9_4, 0.9_7_9_0, 0.9_9_6_4, 0.9_9_7_2, 0.9_8_6_1]).to(__a)
			lowercase__  :   Tuple    		=      [75, 75, 17, 63, 17]
			lowercase__  :   Any    		=      torch.tensor([3_3_5.0_6_0_9, 7_9.3_8_4_8, 3_7_5.4_2_1_6, 1_8_7.2_4_9_5]).to(__a)

			self.assertEqual(len(results["""scores"""])					,  5)
			self.assertTrue(torch.allclose(results["""scores"""]					,  __a					,  atol=1E-4))
			self.assertSequenceEqual(results["""labels"""].tolist()					,  __a)
			self.assertTrue(torch.allclose(results["""boxes"""][0, :]					,  __a))








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCamelCase__    :		Dict          =   """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCamelCase__    :		Tuple          =   """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCamelCase__    :		Any          =   """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Optional[Any]:


  '''simple docstring'''






  return float((preds == labels).mean()   )

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_="binary"   )		->	int:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      simple_accuracy(lowercase_ , lowercase_   )
  lowercase__  :   Optional[Any]    		=      float(fa_score(y_true=lowercase_ , y_pred=lowercase_ , average=lowercase_   )   )
  return {
      "accuracy": acc,
      "f1": fa,
  }

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[Any]    		=      {}
  for id_pred, label in zip(lowercase_ , lowercase_   ):
    lowercase__  :   Optional[Any]    		=      F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
    lowercase__  :   Optional[Any]    		=      id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)   )
    else:
      lowercase__  :   Optional[int]    		=      [(pred, label)]
  lowercase__		,    lowercase__  :   List[str]    		=      [], []
  for question, preds_labels in question_map.items():
    lowercase__		,    lowercase__  :   List[str]    		=      zip(*lowercase_   )
    lowercase__  :   Optional[Any]    		=      fa_score(y_true=lowercase_ , y_pred=lowercase_ , average="""macro"""   )
    fas.append(lowercase_   )
    lowercase__  :   List[Any]    		=      int(sum(pred == label for pred, label in preds_labels   ) == len(lowercase_   )   )
    ems.append(lowercase_   )
  lowercase__  :   str    		=      float(sum(lowercase_   ) / len(lowercase_   )   )
  lowercase__  :   str    		=      sum(lowercase_   ) / len(lowercase_   )
  lowercase__  :   List[str]    		=      float(fa_score(y_true=lowercase_ , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION							)
class       _snake_case   (	datasets.Metric							):


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name not in [
       "boolq",
       "cb",
       "copa",
       "multirc",
       "record",
       "rte",
       "wic",
       "wsc",
       "wsc.fixed",
       "axb",
       "axg",
   ]:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
   return datasets.MetricInfo(
       description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(self._get_feature_types())					,  codebase_urls=[]					,  reference_urls=[]					,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name == "record":
     return {
         "predictions": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "prediction_text": datasets.Value("""string"""),
         },
         "references": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "answers": datasets.Sequence(datasets.Value("""string""")),
         },
     }
   elif self.config_name == "multirc":
     return {
         "predictions": {
             "idx": {
                 "answer": datasets.Value("""int64"""),
                 "paragraph": datasets.Value("""int64"""),
                 "question": datasets.Value("""int64"""),
             },
             "prediction": datasets.Value("""int64"""),
         },
         "references": datasets.Value("""int64"""),
     }
   else:
     return {
         "predictions": datasets.Value("""int64"""),
         "references": datasets.Value("""int64"""),
     }



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.config_name == "axb":
     return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   elif self.config_name == "cb":
     return acc_and_fa(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  fa_avg="""macro""")
   elif self.config_name == "record":
     lowercase__  :   Optional[Any]    		=      [
         {
             """qas""": [
                 {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                 for ref in references
             ]
         }
     ]
     lowercase__  :   str    		=      {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
     return evaluate_record(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)[0]
   elif self.config_name == "multirc":
     return evaluate_multirc(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
     return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   else:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






