
import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin



class       _snake_case   (	UpperCAmelCase_  ,						unittest.TestCase							):
 __lowerCAmelCase  :     Optional[Any]        =			ProphetNetTokenizer
 __lowerCAmelCase  :     Union[str, Any]        =			False


 def      lowercase__     (					self):


   '''simple docstring'''




   super().setUp()

   lowercase__  :   Tuple    		=      [
       """[UNK]""",
       """[CLS]""",
       """[SEP]""",
       """[PAD]""",
       """[MASK]""",
       """want""",
       """##want""",
       """##ed""",
       """wa""",
       """un""",
       """runn""",
       """##ing""",
       """,""",
       """low""",
       """lowest""",
   ]
   lowercase__  :   Optional[Any]    		=      os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES["""vocab_file"""])
   with open(self.vocab_file					,  """w"""					,  encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   List[Any]    		=      """UNwant\u00E9d,running"""
   lowercase__  :   Tuple    		=      """unwanted, running"""
   return input_text, output_text


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Union[str, Any]    		=      self.tokenizer_class(self.vocab_file)

   lowercase__  :   Dict    		=      tokenizer.tokenize("""UNwant\u00E9d,running""")
   self.assertListEqual(SCREAMING_SNAKE_CASE_					,  ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""])
   self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)					,  [9, 6, 7, 12, 10, 11])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Any    		=      BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")					,  ["""ah""", """\u535A""", """\u63A8""", """zz"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Union[str, Any]    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)					,  ["""hello""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")					,  ["""hello"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Union[str, Any]    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,  strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)					,  ["""hällo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")					,  ["""h\u00E9llo"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Any    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,  strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)					,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")					,  ["""hello"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Optional[Any]    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)					,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")					,  ["""hello"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Dict    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)					,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Optional[int]    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,  strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)					,  ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   str    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,  strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)					,  ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Tuple    		=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_					,  never_split=["""[UNK]"""])

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")					,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])


 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Optional[Any]    		=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

   lowercase__  :   List[str]    		=      {}
   for i, token in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__  :   Union[str, Any]    		=      i
   lowercase__  :   Optional[Any]    		=      WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_					,  unk_token="""[UNK]""")

   self.assertListEqual(tokenizer.tokenize("""""")					,  [])

   self.assertListEqual(tokenizer.tokenize("""unwanted running""")					,  ["""un""", """##want""", """##ed""", """runn""", """##ing"""])

   self.assertListEqual(tokenizer.tokenize("""unwantedX running""")					,  ["""[UNK]""", """runn""", """##ing"""])


 @require_torch
 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   List[Any]    		=      self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased""")

   lowercase__  :   Optional[int]    		=      ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
   lowercase__  :   Tuple    		=      [10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
   lowercase__  :   Union[str, Any]    		=      tokenizer(SCREAMING_SNAKE_CASE_					,  padding=SCREAMING_SNAKE_CASE_					,  return_tensors="""pt""")
   self.assertIsInstance(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   lowercase__  :   int    		=      list(batch.input_ids.numpy()[0])
   self.assertListEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   self.assertEqual((2, 9)					,  batch.input_ids.shape)
   self.assertEqual((2, 9)					,  batch.attention_mask.shape)


 def      lowercase__     (					self):


   '''simple docstring'''




   self.assertTrue(_is_whitespace(""" """))
   self.assertTrue(_is_whitespace("""\t"""))
   self.assertTrue(_is_whitespace("""\r"""))
   self.assertTrue(_is_whitespace("""\n"""))
   self.assertTrue(_is_whitespace("""\u00A0"""))

   self.assertFalse(_is_whitespace("""A"""))
   self.assertFalse(_is_whitespace("""-"""))


 def      lowercase__     (					self):


   '''simple docstring'''




   self.assertTrue(_is_control("""\u0005"""))

   self.assertFalse(_is_control("""A"""))
   self.assertFalse(_is_control(""" """))
   self.assertFalse(_is_control("""\t"""))
   self.assertFalse(_is_control("""\r"""))


 def      lowercase__     (					self):


   '''simple docstring'''




   self.assertTrue(_is_punctuation("""-"""))
   self.assertTrue(_is_punctuation("""$"""))
   self.assertTrue(_is_punctuation("""`"""))
   self.assertTrue(_is_punctuation("""."""))

   self.assertFalse(_is_punctuation("""A"""))
   self.assertFalse(_is_punctuation(""" """))



 @slow
 def      lowercase__     (					self):


   '''simple docstring'''




   lowercase__  :   Any    		=      self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased""")

   lowercase__  :   List[Any]    		=      tokenizer.encode("""sequence builders"""					,  add_special_tokens=SCREAMING_SNAKE_CASE_)
   lowercase__  :   List[str]    		=      tokenizer.encode("""multi-sequence build"""					,  add_special_tokens=SCREAMING_SNAKE_CASE_)

   lowercase__  :   List[Any]    		=      tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_)
   lowercase__  :   Optional[int]    		=      tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)

   assert encoded_sentence == text + [1_02]
   assert encoded_pair == text + [1_02] + text_a + [1_02]








from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCamelCase__    :		Dict          =   """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCamelCase__    :		Tuple          =   """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCamelCase__    :		Any          =   """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Optional[Any]:


  '''simple docstring'''






  return float((preds == labels).mean()   )

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_="binary"   )		->	int:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      simple_accuracy(lowercase_ , lowercase_   )
  lowercase__  :   Optional[Any]    		=      float(fa_score(y_true=lowercase_ , y_pred=lowercase_ , average=lowercase_   )   )
  return {
      "accuracy": acc,
      "f1": fa,
  }

def 		UpperCamelCase	(      lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   List[Any]    		=      {}
  for id_pred, label in zip(lowercase_ , lowercase_   ):
    lowercase__  :   Optional[Any]    		=      F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
    lowercase__  :   Optional[Any]    		=      id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)   )
    else:
      lowercase__  :   Optional[int]    		=      [(pred, label)]
  lowercase__		,    lowercase__  :   List[str]    		=      [], []
  for question, preds_labels in question_map.items():
    lowercase__		,    lowercase__  :   List[str]    		=      zip(*lowercase_   )
    lowercase__  :   Optional[Any]    		=      fa_score(y_true=lowercase_ , y_pred=lowercase_ , average="""macro"""   )
    fas.append(lowercase_   )
    lowercase__  :   List[Any]    		=      int(sum(pred == label for pred, label in preds_labels   ) == len(lowercase_   )   )
    ems.append(lowercase_   )
  lowercase__  :   str    		=      float(sum(lowercase_   ) / len(lowercase_   )   )
  lowercase__  :   str    		=      sum(lowercase_   ) / len(lowercase_   )
  lowercase__  :   List[str]    		=      float(fa_score(y_true=lowercase_ , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]   )   )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION							)
class       _snake_case   (	datasets.Metric							):


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name not in [
       "boolq",
       "cb",
       "copa",
       "multirc",
       "record",
       "rte",
       "wic",
       "wsc",
       "wsc.fixed",
       "axb",
       "axg",
   ]:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
   return datasets.MetricInfo(
       description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(self._get_feature_types())					,  codebase_urls=[]					,  reference_urls=[]					,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None					,  )


 def      lowercase__     (					self):


   '''simple docstring'''




   if self.config_name == "record":
     return {
         "predictions": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "prediction_text": datasets.Value("""string"""),
         },
         "references": {
             "idx": {
                 "passage": datasets.Value("""int64"""),
                 "query": datasets.Value("""int64"""),
             },
             "answers": datasets.Sequence(datasets.Value("""string""")),
         },
     }
   elif self.config_name == "multirc":
     return {
         "predictions": {
             "idx": {
                 "answer": datasets.Value("""int64"""),
                 "paragraph": datasets.Value("""int64"""),
                 "question": datasets.Value("""int64"""),
             },
             "prediction": datasets.Value("""int64"""),
         },
         "references": datasets.Value("""int64"""),
     }
   else:
     return {
         "predictions": datasets.Value("""int64"""),
         "references": datasets.Value("""int64"""),
     }



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.config_name == "axb":
     return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   elif self.config_name == "cb":
     return acc_and_fa(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  fa_avg="""macro""")
   elif self.config_name == "record":
     lowercase__  :   Optional[Any]    		=      [
         {
             """qas""": [
                 {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                 for ref in references
             ]
         }
     ]
     lowercase__  :   str    		=      {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
     return evaluate_record(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)[0]
   elif self.config_name == "multirc":
     return evaluate_multirc(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
   elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
     return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)}
   else:
     raise KeyError(
         """You should supply a configuration name selected in """
         """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






