
from importlib import import_module

from .logging import get_logger


lowerCamelCase__    :		Optional[Any]          =   get_logger(__name__)



class       _snake_case   :


	def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=None):


			'''simple docstring'''




			lowercase__  :   int    		=      attrs or []
			if module is not None:
					for key in module.__dict__:
							if key in attrs or not key.startswith("""__"""):
									setattr(self					,  SCREAMING_SNAKE_CASE_					,  getattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_))
			lowercase__  :   str    		=      module._original_module if isinstance(SCREAMING_SNAKE_CASE_					,  _PatchedModuleObj) else module



class       _snake_case   :
	__lowerCAmelCase  :     Optional[int]        =			[]


	def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_=None):


			'''simple docstring'''




			lowercase__  :   Union[str, Any]    		=      obj
			lowercase__  :   str    		=      target
			lowercase__  :   Dict    		=      new
			lowercase__  :   Optional[int]    		=      target.split(""".""")[0]
			lowercase__  :   Dict    		=      {}
			lowercase__  :   Dict    		=      attrs or []


	def __enter__(					self):


			'''simple docstring'''




			lowercase__  :   Optional[Any]    		=      self.target.split(""".""")

			# Patch modules:
			# it's used to patch attributes of submodules like "os.path.join";
			# in this case we need to patch "os" and "os.path"

			for i in range(len(SCREAMING_SNAKE_CASE_)):
					try:
							lowercase__  :   List[Any]    		=      import_module(""".""".join(submodules[: i + 1]))
					except ModuleNotFoundError:
							continue
					# We iterate over all the globals in self.obj in case we find "os" or "os.path"
					for attr in self.obj.__dir__():
							lowercase__  :   Dict    		=      getattr(self.obj					,  SCREAMING_SNAKE_CASE_)
							# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
							# This allows to patch renamed modules like "from os import path as ospath".
							if obj_attr is submodule or (
							    (isinstance(SCREAMING_SNAKE_CASE_					,  _PatchedModuleObj) and obj_attr._original_module is submodule)
							):
									lowercase__  :   Optional[Any]    		=      obj_attr
									# patch at top level
									setattr(self.obj					,  SCREAMING_SNAKE_CASE_					,  _PatchedModuleObj(SCREAMING_SNAKE_CASE_					,  attrs=self.attrs))
									lowercase__  :   Optional[int]    		=      getattr(self.obj					,  SCREAMING_SNAKE_CASE_)
									# construct lower levels patches
									for key in submodules[i + 1 :]:
											setattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  _PatchedModuleObj(getattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)					,  attrs=self.attrs))
											lowercase__  :   str    		=      getattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
									# finally set the target attribute
									setattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  self.new)

							 # Patch attribute itself:
							 # it's used for builtins like "open",
							 # and also to patch "os.path.join" we may also need to patch "join"
							 # itself if it was imported as "from os.path import join".

			if submodules:  # if it's an attribute of a submodule like "os.path.join"
					try:
							lowercase__  :   Union[str, Any]    		=      getattr(import_module(""".""".join(SCREAMING_SNAKE_CASE_))					,  SCREAMING_SNAKE_CASE_)
					except (AttributeError, ModuleNotFoundError):
							return
					# We iterate over all the globals in self.obj in case we find "os.path.join"
					for attr in self.obj.__dir__():
							# We don't check for the name of the global, but rather if its value *is* "os.path.join".
							# This allows to patch renamed attributes like "from os.path import join as pjoin".
							if getattr(self.obj					,  SCREAMING_SNAKE_CASE_) is attr_value:
									lowercase__  :   str    		=      getattr(self.obj					,  SCREAMING_SNAKE_CASE_)
									setattr(self.obj					,  SCREAMING_SNAKE_CASE_					,  self.new)
			elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
					lowercase__  :   Optional[int]    		=      globals()["""__builtins__"""][target_attr]
					setattr(self.obj					,  SCREAMING_SNAKE_CASE_					,  self.new)
			else:
					raise RuntimeError(f'Tried to patch attribute {target_attr} instead of a submodule.')


	def __exit__(					self					,  *SCREAMING_SNAKE_CASE_):


			'''simple docstring'''




			for attr in list(self.original):
					setattr(self.obj					,  SCREAMING_SNAKE_CASE_					,  self.original.pop(SCREAMING_SNAKE_CASE_))


	def      lowercase__     (					self):


			'''simple docstring'''




			self.__enter__()
			self._active_patches.append(self)



	def      lowercase__     (					self):


			'''simple docstring'''




			try:
					self._active_patches.remove(self)
			except ValueError:
					# If the patch hasn't been started this will fail
					return None

			return self.__exit__()








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCamelCase__    :		Dict          =   """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCamelCase__    :		Tuple          =   """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCamelCase__    :		Any          =   """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Optional[Any]:


  '''simple docstring'''






  return float((preds == labels).mean()   )

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_="binary"   )		->	int:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      simple_accuracy(lowercase_ , lowercase_   )
  lowercase__  :   Optional[Any]    		=      float(fa_score(y_true=lowercase_ , y_pred=lowercase_ , average=lowercase_   )   )
  return {
      "accuracy": acc,
      "f1": fa,
  }

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[Any]    		=      {}
  for id_pred, label in zip(lowercase_ , lowercase_   ):
    lowercase__  :   Optional[Any]    		=      F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
    lowercase__  :   Optional[Any]    		=      id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)   )
    else:
      lowercase__  :   Optional[int]    		=      [(pred, label)]
  lowercase__		,    lowercase__  :   List[str]    		=      [], []
  for question, preds_labels in question_map.items():
    lowercase__		,    lowercase__  :   List[str]    		=      zip(*lowercase_   )
    lowercase__  :   Optional[Any]    		=      fa_score(y_true=lowercase_ , y_pred=lowercase_ , average="""macro"""   )
    fas.append(lowercase_   )
    lowercase__  :   List[Any]    		=      int(sum(pred == label for pred, label in preds_labels   ) == len(lowercase_   )   )
    ems.append(lowercase_   )
  lowercase__  :   str    		=      float(sum(lowercase_   ) / len(lowercase_   )   )
  lowercase__  :   str    		=      sum(lowercase_   ) / len(lowercase_   )
  lowercase__  :   List[str]    		=      float(fa_score(y_true=lowercase_ , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION							)
class       _snake_case   (	datasets.Metric							):


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name not in [
       "boolq",
       "cb",
       "copa",
       "multirc",
       "record",
       "rte",
       "wic",
       "wsc",
       "wsc.fixed",
       "axb",
       "axg",
   ]:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
   return datasets.MetricInfo(
       description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(self._get_feature_types())					,  codebase_urls=[]					,  reference_urls=[]					,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name == "record":
     return {
         "predictions": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "prediction_text": datasets.Value("""string"""),
         },
         "references": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "answers": datasets.Sequence(datasets.Value("""string""")),
         },
     }
   elif self.config_name == "multirc":
     return {
         "predictions": {
             "idx": {
                 "answer": datasets.Value("""int64"""),
                 "paragraph": datasets.Value("""int64"""),
                 "question": datasets.Value("""int64"""),
             },
             "prediction": datasets.Value("""int64"""),
         },
         "references": datasets.Value("""int64"""),
     }
   else:
     return {
         "predictions": datasets.Value("""int64"""),
         "references": datasets.Value("""int64"""),
     }



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.config_name == "axb":
     return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   elif self.config_name == "cb":
     return acc_and_fa(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  fa_avg="""macro""")
   elif self.config_name == "record":
     lowercase__  :   Optional[Any]    		=      [
         {
             """qas""": [
                 {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                 for ref in references
             ]
         }
     ]
     lowercase__  :   str    		=      {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
     return evaluate_record(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)[0]
   elif self.config_name == "multirc":
     return evaluate_multirc(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
     return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   else:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






