

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_SCREAMING_SNAKE_CASE  :						Union[str, Any]         =    re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

_SCREAMING_SNAKE_CASE  :						List[str]         =    None
def        UpperCAmelCase_     (		):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''       )
     parser.add_argument('''data_file'''   ,	metavar='''data.json'''   ,	help='''Input data JSON file.'''       )
     parser.add_argument('''pred_file'''   ,	metavar='''pred.json'''   ,	help='''Model predictions.'''       )
     parser.add_argument(
         '''--out-file'''   ,	'''-o'''   ,	metavar='''eval.json'''   ,	help='''Write accuracy metrics to file (default is stdout).'''       )
     parser.add_argument(
         '''--na-prob-file'''   ,	'''-n'''   ,	metavar='''na_prob.json'''   ,	help='''Model estimates of probability of no answer.'''       )
     parser.add_argument(
         '''--na-prob-thresh'''   ,	'''-t'''   ,	type=_A   ,	default=1.0   ,	help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''   ,	)
     parser.add_argument(
         '''--out-image-dir'''   ,	'''-p'''   ,	metavar='''out_images'''   ,	default=_A   ,	help='''Save precision-recall curves to directory.'''       )
     parser.add_argument('''--verbose'''   ,	'''-v'''   ,	action='''store_true'''       )
     if len(sys.argv       ) == 1:
          parser.print_help()
          sys.exit(1       )
     return parser.parse_args()
def        UpperCAmelCase_     (		_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    SCREAMING_SNAKE_CASE__					=					bool(qa['''answers''']['''text''']       )
     return qid_to_has_ans
def        UpperCAmelCase_     (		_A       ):





     '''simple docstring'''




     def remove_articles(_A       ):
          return ARTICLES_REGEX.sub(''' '''   ,	_A       )

     def white_space_fix(_A       ):
          return " ".join(text.split()       )

     def remove_punc(_A       ):
          SCREAMING_SNAKE_CASE__					=					set(string.punctuation       )
          return "".join(ch for ch in text if ch not in exclude       )

     def lower(_A       ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(_A       )       )       )       )
def        UpperCAmelCase_     (		_A       ):





     '''simple docstring'''




     if not s:
          return []
     return normalize_answer(_A       ).split()
def        UpperCAmelCase_     (		_A   ,	_A       ):





     '''simple docstring'''




     return int(normalize_answer(_A       ) == normalize_answer(_A       )       )
def        UpperCAmelCase_     (		_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					get_tokens(_A       )
     SCREAMING_SNAKE_CASE__					=					get_tokens(_A       )
     SCREAMING_SNAKE_CASE__					=					collections.Counter(_A       ) & collections.Counter(_A       )
     SCREAMING_SNAKE_CASE__					=					sum(common.values()       )
     if len(_A       ) == 0 or len(_A       ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks       )
     if num_same == 0:
          return 0
     SCREAMING_SNAKE_CASE__					=					1.0 * num_same / len(_A       )
     SCREAMING_SNAKE_CASE__					=					1.0 * num_same / len(_A       )
     SCREAMING_SNAKE_CASE__					=					(2 * precision * recall) / (precision + recall)
     return fa
def        UpperCAmelCase_     (		_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					{}
     SCREAMING_SNAKE_CASE__					=					{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    SCREAMING_SNAKE_CASE__					=					qa['''id''']
                    SCREAMING_SNAKE_CASE__					=					[t for t in qa['''answers''']['''text'''] if normalize_answer(_A       )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         SCREAMING_SNAKE_CASE__					=					['''''']
                    if qid not in preds:
                         print(F'''Missing prediction for {qid}'''       )
                         continue
                    SCREAMING_SNAKE_CASE__					=					preds[qid]
                    # Take max over all gold answers
                    SCREAMING_SNAKE_CASE__					=					max(compute_exact(_A   ,	_A       ) for a in gold_answers       )
                    SCREAMING_SNAKE_CASE__					=					max(compute_fa(_A   ,	_A       ) for a in gold_answers       )
     return exact_scores, fa_scores
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					{}
     for qid, s in scores.items():
          SCREAMING_SNAKE_CASE__					=					na_probs[qid] > na_prob_thresh
          if pred_na:
               SCREAMING_SNAKE_CASE__					=					float(not qid_to_has_ans[qid]       )
          else:
               SCREAMING_SNAKE_CASE__					=					s
     return new_scores
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A=None       ):





     '''simple docstring'''




     if not qid_list:
          SCREAMING_SNAKE_CASE__					=					len(_A       )
          return collections.OrderedDict(
              [
                  ('''exact''', 1_0_0.0 * sum(exact_scores.values()       ) / total),
                  ('''f1''', 1_0_0.0 * sum(fa_scores.values()       ) / total),
                  ('''total''', total),
              ]       )
     else:
          SCREAMING_SNAKE_CASE__					=					len(_A       )
          return collections.OrderedDict(
              [
                  ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list       ) / total),
                  ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list       ) / total),
                  ('''total''', total),
              ]       )
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     for k in new_eval:
          SCREAMING_SNAKE_CASE__					=					new_eval[k]
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     plt.step(_A   ,	_A   ,	color='''b'''   ,	alpha=0.2   ,	where='''post'''       )
     plt.fill_between(_A   ,	_A   ,	step='''post'''   ,	alpha=0.2   ,	color='''b'''       )
     plt.xlabel('''Recall'''       )
     plt.ylabel('''Precision'''       )
     plt.xlim([0.0, 1.0_5]       )
     plt.ylim([0.0, 1.0_5]       )
     plt.title(_A       )
     plt.savefig(_A       )
     plt.clf()
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A   ,	_A=None   ,	_A=None       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					sorted(_A   ,	key=lambda _A       : na_probs[k]       )
     SCREAMING_SNAKE_CASE__					=					0.0
     SCREAMING_SNAKE_CASE__					=					1.0
     SCREAMING_SNAKE_CASE__					=					0.0
     SCREAMING_SNAKE_CASE__					=					[1.0]
     SCREAMING_SNAKE_CASE__					=					[0.0]
     SCREAMING_SNAKE_CASE__					=					0.0
     for i, qid in enumerate(_A       ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          SCREAMING_SNAKE_CASE__					=					true_pos / float(i + 1       )
          SCREAMING_SNAKE_CASE__					=					true_pos / float(_A       )
          if i == len(_A       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(_A       )
               recalls.append(_A       )
     if out_image:
          plot_pr_curve(_A   ,	_A   ,	_A   ,	_A       )
     return {"ap": 1_0_0.0 * avg_prec}
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     if out_image_dir and not os.path.exists(_A       ):
          os.makedirs(_A       )
     SCREAMING_SNAKE_CASE__					=					sum(1 for v in qid_to_has_ans.values() if v       )
     if num_true_pos == 0:
          return
     SCREAMING_SNAKE_CASE__					=					make_precision_recall_eval(
         _A   ,	_A   ,	_A   ,	_A   ,	out_image=os.path.join(_A   ,	'''pr_exact.png'''       )   ,	title='''Precision-Recall curve for Exact Match score'''   ,	)
     SCREAMING_SNAKE_CASE__					=					make_precision_recall_eval(
         _A   ,	_A   ,	_A   ,	_A   ,	out_image=os.path.join(_A   ,	'''pr_f1.png'''       )   ,	title='''Precision-Recall curve for F1 score'''   ,	)
     SCREAMING_SNAKE_CASE__					=					{k: float(_A       ) for k, v in qid_to_has_ans.items()}
     SCREAMING_SNAKE_CASE__					=					make_precision_recall_eval(
         _A   ,	_A   ,	_A   ,	_A   ,	out_image=os.path.join(_A   ,	'''pr_oracle.png'''       )   ,	title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''   ,	)
     merge_eval(_A   ,	_A   ,	'''pr_exact'''       )
     merge_eval(_A   ,	_A   ,	'''pr_f1'''       )
     merge_eval(_A   ,	_A   ,	'''pr_oracle'''       )
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     if not qid_list:
          return
     SCREAMING_SNAKE_CASE__					=					[na_probs[k] for k in qid_list]
     SCREAMING_SNAKE_CASE__					=					np.ones_like(_A       ) / float(len(_A       )       )
     plt.hist(_A   ,	weights=_A   ,	bins=20   ,	range=(0.0, 1.0)       )
     plt.xlabel('''Model probability of no-answer'''       )
     plt.ylabel('''Proportion of dataset'''       )
     plt.title(F'''Histogram of no-answer probability: {name}'''       )
     plt.savefig(os.path.join(_A   ,	F'''na_prob_hist_{name}.png'''       )       )
     plt.clf()
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
     SCREAMING_SNAKE_CASE__					=					num_no_ans
     SCREAMING_SNAKE_CASE__					=					cur_score
     SCREAMING_SNAKE_CASE__					=					0.0
     SCREAMING_SNAKE_CASE__					=					sorted(_A   ,	key=lambda _A       : na_probs[k]       )
     for i, qid in enumerate(_A       ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               SCREAMING_SNAKE_CASE__					=					scores[qid]
          else:
               if preds[qid]:
                    SCREAMING_SNAKE_CASE__					=					-1
               else:
                    SCREAMING_SNAKE_CASE__					=					0
          cur_score += diff
          if cur_score > best_score:
               SCREAMING_SNAKE_CASE__					=					cur_score
               SCREAMING_SNAKE_CASE__					=					na_probs[qid]
     return 1_0_0.0 * best_score / len(_A       ), best_thresh
def        UpperCAmelCase_     (		_A   ,	_A   ,	_A   ,	_A   ,	_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__,SCREAMING_SNAKE_CASE__					=					find_best_thresh(_A   ,	_A   ,	_A   ,	_A       )
     SCREAMING_SNAKE_CASE__,SCREAMING_SNAKE_CASE__					=					find_best_thresh(_A   ,	_A   ,	_A   ,	_A       )
     SCREAMING_SNAKE_CASE__					=					best_exact
     SCREAMING_SNAKE_CASE__					=					exact_thresh
     SCREAMING_SNAKE_CASE__					=					best_fa
     SCREAMING_SNAKE_CASE__					=					fa_thresh
def        UpperCAmelCase_     (		):





     '''simple docstring'''




     with open(OPTS.data_file       ) as f:
          SCREAMING_SNAKE_CASE__					=					json.load(_A       )
          SCREAMING_SNAKE_CASE__					=					dataset_json['''data''']
     with open(OPTS.pred_file       ) as f:
          SCREAMING_SNAKE_CASE__					=					json.load(_A       )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file       ) as f:
               SCREAMING_SNAKE_CASE__					=					json.load(_A       )
     else:
          SCREAMING_SNAKE_CASE__					=					{k: 0.0 for k in preds}
     SCREAMING_SNAKE_CASE__					=					make_qid_to_has_ans(_A       )  # maps qid to True/False
     SCREAMING_SNAKE_CASE__					=					[k for k, v in qid_to_has_ans.items() if v]
     SCREAMING_SNAKE_CASE__					=					[k for k, v in qid_to_has_ans.items() if not v]
     SCREAMING_SNAKE_CASE__,SCREAMING_SNAKE_CASE__					=					get_raw_scores(_A   ,	_A       )
     SCREAMING_SNAKE_CASE__					=					apply_no_ans_threshold(_A   ,	_A   ,	_A   ,	OPTS.na_prob_thresh       )
     SCREAMING_SNAKE_CASE__					=					apply_no_ans_threshold(_A   ,	_A   ,	_A   ,	OPTS.na_prob_thresh       )
     SCREAMING_SNAKE_CASE__					=					make_eval_dict(_A   ,	_A       )
     if has_ans_qids:
          SCREAMING_SNAKE_CASE__					=					make_eval_dict(_A   ,	_A   ,	qid_list=_A       )
          merge_eval(_A   ,	_A   ,	'''HasAns'''       )
     if no_ans_qids:
          SCREAMING_SNAKE_CASE__					=					make_eval_dict(_A   ,	_A   ,	qid_list=_A       )
          merge_eval(_A   ,	_A   ,	'''NoAns'''       )
     if OPTS.na_prob_file:
          find_all_best_thresh(_A   ,	_A   ,	_A   ,	_A   ,	_A   ,	_A       )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(_A   ,	_A   ,	_A   ,	_A   ,	_A   ,	OPTS.out_image_dir       )
          histogram_na_prob(_A   ,	_A   ,	OPTS.out_image_dir   ,	'''hasAns'''       )
          histogram_na_prob(_A   ,	_A   ,	OPTS.out_image_dir   ,	'''noAns'''       )
     if OPTS.out_file:
          with open(OPTS.out_file   ,	'''w'''       ) as f:
               json.dump(_A   ,	_A       )
     else:
          print(json.dumps(_A   ,	indent=2       )       )


if __name__ == "__main__":
       _SCREAMING_SNAKE_CASE  :						Tuple         =    parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('''Agg''')
              import matplotlib.pyplot as plt
       main()





def        UpperCAmelCase_     (		_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					[]
     for data in source_data:
          for i, el in enumerate(_A       ):
               if len(_A       ) < i + 1:
                    data_lists.append([]       )
               data_lists[i].append(float(_A       )       )
     return data_lists
def        UpperCAmelCase_     (		_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					[]
     for dlist, weight in zip(_A   ,	_A       ):
          SCREAMING_SNAKE_CASE__					=					min(_A       )
          SCREAMING_SNAKE_CASE__					=					max(_A       )

          SCREAMING_SNAKE_CASE__					=					[]
          # for weight 0 score is 1 - actual score
          if weight == 0:
               for item in dlist:
                    try:
                         score.append(1 - ((item - mind) / (maxd - mind))       )
                    except ZeroDivisionError:
                         score.append(1       )

          elif weight == 1:
               for item in dlist:
                    try:
                         score.append((item - mind) / (maxd - mind)       )
                    except ZeroDivisionError:
                         score.append(0       )

        # weight not 0 or 1
          else:
               SCREAMING_SNAKE_CASE__					=					F'''Invalid weight of {weight:f} provided'''
               raise ValueError(_A       )

          score_lists.append(_A       )

     return score_lists
def        UpperCAmelCase_     (		_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					[0 for i in range(len(score_lists[0]       )       )]

     for slist in score_lists:
          for j, ele in enumerate(_A       ):
               SCREAMING_SNAKE_CASE__					=					final_scores[j] + ele

     return final_scores







def        UpperCAmelCase_     (		_A   ,	_A       ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__					=					get_data(_A       )
     SCREAMING_SNAKE_CASE__					=					calculate_each_score(_A   ,	_A       )
     SCREAMING_SNAKE_CASE__					=					generate_final_scores(_A       )

     # append scores to source data
     for i, ele in enumerate(_A       ):
          source_data[i].append(_A       )

     return source_data


