


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__UpperCamelCase    :						int     =							re.compile(R'\b(a|an|the)\b', re.UNICODE)

__UpperCamelCase    :						Any     =							None

def   _UpperCAmelCase       (     ):
      """simple docstring"""




      __lowerCamelCase     :      Union[str, Any]       						=						argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""      )
      parser.add_argument("""data_file"""						,	metavar="""data.json"""						,	help="""Input data JSON file."""      )
      parser.add_argument("""pred_file"""						,	metavar="""pred.json"""						,	help="""Model predictions."""      )
      parser.add_argument(
          """--out-file"""						,	"""-o"""						,	metavar="""eval.json"""						,	help="""Write accuracy metrics to file (default is stdout)."""      )
      parser.add_argument(
          """--na-prob-file"""						,	"""-n"""						,	metavar="""na_prob.json"""						,	help="""Model estimates of probability of no answer."""      )
      parser.add_argument(
          """--na-prob-thresh"""						,	"""-t"""						,	type=SCREAMING_SNAKE_CASE_						,	default=1.0						,	help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""						,	)
      parser.add_argument(
          """--out-image-dir"""						,	"""-p"""						,	metavar="""out_images"""						,	default=SCREAMING_SNAKE_CASE_						,	help="""Save precision-recall curves to directory."""      )
      parser.add_argument("""--verbose"""						,	"""-v"""						,	action="""store_true"""      )
      if len(sys.argv      ) == 1:
            parser.print_help()
            sys.exit(1      )
      return parser.parse_args()

def   _UpperCAmelCase       (     UpperCAmelCase      :			List[str]      ):
      """simple docstring"""




      __lowerCamelCase     :      Optional[int]       						=						{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        __lowerCamelCase     :      Union[str, Any]       						=						bool(qa["""answers"""]["""text"""]      )
      return qid_to_has_ans

def   _UpperCAmelCase       (     UpperCAmelCase      :			int      ):
      """simple docstring"""




      def remove_articles(UpperCAmelCase      :			int      ):
            return ARTICLES_REGEX.sub(""" """						,	SCREAMING_SNAKE_CASE_      )

      def white_space_fix(UpperCAmelCase      :			Tuple      ):
            return " ".join(text.split()      )

      def remove_punc(UpperCAmelCase      :			Dict      ):
            __lowerCamelCase     :      Dict       						=						set(string.punctuation      )
            return "".join(ch for ch in text if ch not in exclude      )

      def lower(UpperCAmelCase      :			str      ):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE_      )      )      )      )

def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[Any]      ):
      """simple docstring"""




      if not s:
            return []
      return normalize_answer(SCREAMING_SNAKE_CASE_      ).split()

def   _UpperCAmelCase       (     UpperCAmelCase      :			Tuple						,	UpperCAmelCase      :			List[Any]      ):
      """simple docstring"""




      return int(normalize_answer(SCREAMING_SNAKE_CASE_      ) == normalize_answer(SCREAMING_SNAKE_CASE_      )      )

def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[int]						,	UpperCAmelCase      :			Optional[Any]      ):
      """simple docstring"""




      __lowerCamelCase     :      List[str]       						=						get_tokens(SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      Union[str, Any]       						=						get_tokens(SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      Optional[Any]       						=						collections.Counter(SCREAMING_SNAKE_CASE_      ) & collections.Counter(SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      List[str]       						=						sum(common.values()      )
      if len(SCREAMING_SNAKE_CASE_      ) == 0 or len(SCREAMING_SNAKE_CASE_      ) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks      )
      if num_same == 0:
            return 0
      __lowerCamelCase     :      List[str]       						=						1.0 * num_same / len(SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      Tuple       						=						1.0 * num_same / len(SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      List[str]       						=						(2 * precision * recall) / (precision + recall)
      return fa

def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[int]						,	UpperCAmelCase      :			Union[str, Any]      ):
      """simple docstring"""




      __lowerCamelCase     :      List[Any]       						=						{}
      __lowerCamelCase     :      List[str]       						=						{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        __lowerCamelCase     :      str       						=						qa["""id"""]
                        __lowerCamelCase     :      Any       						=						[t for t in qa["""answers"""]["""text"""] if normalize_answer(SCREAMING_SNAKE_CASE_      )]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              __lowerCamelCase     :      Dict       						=						[""""""]
                        if qid not in preds:
                              print(f"""Missing prediction for {qid}"""      )
                              continue
                        __lowerCamelCase     :      Tuple       						=						preds[qid]
                        # Take max over all gold answers
                        __lowerCamelCase     :      Any       						=						max(compute_exact(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      ) for a in gold_answers      )
                        __lowerCamelCase     :      Optional[Any]       						=						max(compute_fa(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      ) for a in gold_answers      )
      return exact_scores, fa_scores

def   _UpperCAmelCase       (     UpperCAmelCase      :			Dict						,	UpperCAmelCase      :			Any						,	UpperCAmelCase      :			Union[str, Any]						,	UpperCAmelCase      :			int      ):
      """simple docstring"""




      __lowerCamelCase     :      Any       						=						{}
      for qid, s in scores.items():
            __lowerCamelCase     :      int       						=						na_probs[qid] > na_prob_thresh
            if pred_na:
                  __lowerCamelCase     :      Optional[int]       						=						float(not qid_to_has_ans[qid]      )
            else:
                  __lowerCamelCase     :      Dict       						=						s
      return new_scores

def   _UpperCAmelCase       (     UpperCAmelCase      :			Tuple						,	UpperCAmelCase      :			Optional[Any]						,	UpperCAmelCase      :			Optional[Any]=None      ):
      """simple docstring"""




      if not qid_list:
            __lowerCamelCase     :      Any       						=						len(SCREAMING_SNAKE_CASE_      )
            return collections.OrderedDict(
                [
                    ("""exact""", 1_0_0.0 * sum(exact_scores.values()      ) / total),
                    ("""f1""", 1_0_0.0 * sum(fa_scores.values()      ) / total),
                    ("""total""", total),
                ]      )
      else:
            __lowerCamelCase     :      Any       						=						len(SCREAMING_SNAKE_CASE_      )
            return collections.OrderedDict(
                [
                    ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list      ) / total),
                    ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list      ) / total),
                    ("""total""", total),
                ]      )

def   _UpperCAmelCase       (     UpperCAmelCase      :			Tuple						,	UpperCAmelCase      :			Any						,	UpperCAmelCase      :			List[Any]      ):
      """simple docstring"""




      for k in new_eval:
            __lowerCamelCase     :      Tuple       						=						new_eval[k]

def   _UpperCAmelCase       (     UpperCAmelCase      :			Dict						,	UpperCAmelCase      :			Optional[Any]						,	UpperCAmelCase      :			Optional[Any]						,	UpperCAmelCase      :			Dict      ):
      """simple docstring"""




      plt.step(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	color="""b"""						,	alpha=0.2						,	where="""post"""      )
      plt.fill_between(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	step="""post"""						,	alpha=0.2						,	color="""b"""      )
      plt.xlabel("""Recall"""      )
      plt.ylabel("""Precision"""      )
      plt.xlim([0.0, 1.0_5]      )
      plt.ylim([0.0, 1.0_5]      )
      plt.title(SCREAMING_SNAKE_CASE_      )
      plt.savefig(SCREAMING_SNAKE_CASE_      )
      plt.clf()

def   _UpperCAmelCase       (     UpperCAmelCase      :			int						,	UpperCAmelCase      :			List[str]						,	UpperCAmelCase      :			Any						,	UpperCAmelCase      :			str						,	UpperCAmelCase      :			Optional[Any]=None						,	UpperCAmelCase      :			Optional[Any]=None      ):
      """simple docstring"""




      __lowerCamelCase     :      Any       						=						sorted(SCREAMING_SNAKE_CASE_						,	key=lambda UpperCAmelCase      : na_probs[k]      )
      __lowerCamelCase     :      int       						=						0.0
      __lowerCamelCase     :      Any       						=						1.0
      __lowerCamelCase     :      Union[str, Any]       						=						0.0
      __lowerCamelCase     :      Any       						=						[1.0]
      __lowerCamelCase     :      Union[str, Any]       						=						[0.0]
      __lowerCamelCase     :      str       						=						0.0
      for i, qid in enumerate(SCREAMING_SNAKE_CASE_      ):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            __lowerCamelCase     :      List[str]       						=						true_pos / float(i + 1      )
            __lowerCamelCase     :      str       						=						true_pos / float(SCREAMING_SNAKE_CASE_      )
            if i == len(SCREAMING_SNAKE_CASE_      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(SCREAMING_SNAKE_CASE_      )
                  recalls.append(SCREAMING_SNAKE_CASE_      )
      if out_image:
            plot_pr_curve(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      return {"ap": 1_0_0.0 * avg_prec}

def   _UpperCAmelCase       (     UpperCAmelCase      :			str						,	UpperCAmelCase      :			List[str]						,	UpperCAmelCase      :			List[str]						,	UpperCAmelCase      :			Optional[int]						,	UpperCAmelCase      :			List[Any]						,	UpperCAmelCase      :			Dict      ):
      """simple docstring"""




      if out_image_dir and not os.path.exists(SCREAMING_SNAKE_CASE_      ):
            os.makedirs(SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      int       						=						sum(1 for v in qid_to_has_ans.values() if v      )
      if num_true_pos == 0:
            return
      __lowerCamelCase     :      Any       						=						make_precision_recall_eval(
          SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	out_image=os.path.join(SCREAMING_SNAKE_CASE_						,	"""pr_exact.png"""      )						,	title="""Precision-Recall curve for Exact Match score"""						,	)
      __lowerCamelCase     :      Tuple       						=						make_precision_recall_eval(
          SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	out_image=os.path.join(SCREAMING_SNAKE_CASE_						,	"""pr_f1.png"""      )						,	title="""Precision-Recall curve for F1 score"""						,	)
      __lowerCamelCase     :      Optional[int]       						=						{k: float(SCREAMING_SNAKE_CASE_      ) for k, v in qid_to_has_ans.items()}
      __lowerCamelCase     :      Tuple       						=						make_precision_recall_eval(
          SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	out_image=os.path.join(SCREAMING_SNAKE_CASE_						,	"""pr_oracle.png"""      )						,	title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""						,	)
      merge_eval(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	"""pr_exact"""      )
      merge_eval(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	"""pr_f1"""      )
      merge_eval(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	"""pr_oracle"""      )

def   _UpperCAmelCase       (     UpperCAmelCase      :			List[Any]						,	UpperCAmelCase      :			List[Any]						,	UpperCAmelCase      :			Optional[int]						,	UpperCAmelCase      :			Tuple      ):
      """simple docstring"""




      if not qid_list:
            return
      __lowerCamelCase     :      Union[str, Any]       						=						[na_probs[k] for k in qid_list]
      __lowerCamelCase     :      Optional[int]       						=						np.ones_like(SCREAMING_SNAKE_CASE_      ) / float(len(SCREAMING_SNAKE_CASE_      )      )
      plt.hist(SCREAMING_SNAKE_CASE_						,	weights=SCREAMING_SNAKE_CASE_						,	bins=20						,	range=(0.0, 1.0)      )
      plt.xlabel("""Model probability of no-answer"""      )
      plt.ylabel("""Proportion of dataset"""      )
      plt.title(f"""Histogram of no-answer probability: {name}"""      )
      plt.savefig(os.path.join(SCREAMING_SNAKE_CASE_						,	f"""na_prob_hist_{name}.png"""      )      )
      plt.clf()

def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[int]						,	UpperCAmelCase      :			Tuple						,	UpperCAmelCase      :			Optional[int]						,	UpperCAmelCase      :			str      ):
      """simple docstring"""




      __lowerCamelCase     :      Optional[int]       						=						sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
      __lowerCamelCase     :      Optional[Any]       						=						num_no_ans
      __lowerCamelCase     :      Union[str, Any]       						=						cur_score
      __lowerCamelCase     :      Dict       						=						0.0
      __lowerCamelCase     :      List[str]       						=						sorted(SCREAMING_SNAKE_CASE_						,	key=lambda UpperCAmelCase      : na_probs[k]      )
      for i, qid in enumerate(SCREAMING_SNAKE_CASE_      ):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  __lowerCamelCase     :      str       						=						scores[qid]
            else:
                  if preds[qid]:
                        __lowerCamelCase     :      str       						=						-1
                  else:
                        __lowerCamelCase     :      Tuple       						=						0
            cur_score += diff
            if cur_score > best_score:
                  __lowerCamelCase     :      Optional[Any]       						=						cur_score
                  __lowerCamelCase     :      int       						=						na_probs[qid]
      return 1_0_0.0 * best_score / len(SCREAMING_SNAKE_CASE_      ), best_thresh

def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[Any]						,	UpperCAmelCase      :			Optional[Any]						,	UpperCAmelCase      :			List[Any]						,	UpperCAmelCase      :			Tuple						,	UpperCAmelCase      :			List[str]						,	UpperCAmelCase      :			Dict      ):
      """simple docstring"""




      __lowerCamelCase						, __lowerCamelCase     :      Tuple       						=						find_best_thresh(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase						, __lowerCamelCase     :      Tuple       						=						find_best_thresh(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      Optional[Any]       						=						best_exact
      __lowerCamelCase     :      Dict       						=						exact_thresh
      __lowerCamelCase     :      Optional[int]       						=						best_fa
      __lowerCamelCase     :      Optional[Any]       						=						fa_thresh

def   _UpperCAmelCase       (     ):
      """simple docstring"""




      with open(OPTS.data_file      ) as f:
            __lowerCamelCase     :      Tuple       						=						json.load(SCREAMING_SNAKE_CASE_      )
            __lowerCamelCase     :      Any       						=						dataset_json["""data"""]
      with open(OPTS.pred_file      ) as f:
            __lowerCamelCase     :      int       						=						json.load(SCREAMING_SNAKE_CASE_      )
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file      ) as f:
                  __lowerCamelCase     :      str       						=						json.load(SCREAMING_SNAKE_CASE_      )
      else:
            __lowerCamelCase     :      str       						=						{k: 0.0 for k in preds}
      __lowerCamelCase     :      Tuple       						=						make_qid_to_has_ans(SCREAMING_SNAKE_CASE_      )  # maps qid to True/False
      __lowerCamelCase     :      Optional[Any]       						=						[k for k, v in qid_to_has_ans.items() if v]
      __lowerCamelCase     :      Dict       						=						[k for k, v in qid_to_has_ans.items() if not v]
      __lowerCamelCase						, __lowerCamelCase     :      Dict       						=						get_raw_scores(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      __lowerCamelCase     :      str       						=						apply_no_ans_threshold(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	OPTS.na_prob_thresh      )
      __lowerCamelCase     :      str       						=						apply_no_ans_threshold(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	OPTS.na_prob_thresh      )
      __lowerCamelCase     :      Optional[int]       						=						make_eval_dict(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      if has_ans_qids:
            __lowerCamelCase     :      List[str]       						=						make_eval_dict(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	qid_list=SCREAMING_SNAKE_CASE_      )
            merge_eval(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	"""HasAns"""      )
      if no_ans_qids:
            __lowerCamelCase     :      Optional[Any]       						=						make_eval_dict(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	qid_list=SCREAMING_SNAKE_CASE_      )
            merge_eval(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	"""NoAns"""      )
      if OPTS.na_prob_file:
            find_all_best_thresh(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	OPTS.out_image_dir      )
            histogram_na_prob(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	OPTS.out_image_dir						,	"""hasAns"""      )
            histogram_na_prob(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						,	OPTS.out_image_dir						,	"""noAns"""      )
      if OPTS.out_file:
            with open(OPTS.out_file						,	"""w"""      ) as f:
                  json.dump(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      )
      else:
            print(json.dumps(SCREAMING_SNAKE_CASE_						,	indent=2      )      )


if __name__ == "__main__":
    __UpperCamelCase    :						List[str]     =							parse_args()
    if OPTS.out_image_dir:
        import matplotlib

        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
    main()







def   _UpperCAmelCase       (     ):
      """simple docstring"""




      __lowerCamelCase     :      Optional[int]       						=						[31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

      __lowerCamelCase     :      str       						=						6
      __lowerCamelCase     :      Optional[int]       						=						1
      __lowerCamelCase     :      Optional[int]       						=						1_901

      __lowerCamelCase     :      Optional[Any]       						=						0

      while year < 2_001:
            day += 7

            if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0):
                  if day > days_per_month[month - 1] and month != 2:
                        month += 1
                        __lowerCamelCase     :      Tuple       						=						day - days_per_month[month - 2]
                  elif day > 29 and month == 2:
                        month += 1
                        __lowerCamelCase     :      str       						=						day - 29
            else:
                  if day > days_per_month[month - 1]:
                        month += 1
                        __lowerCamelCase     :      Any       						=						day - days_per_month[month - 2]

            if month > 12:
                  year += 1
                  __lowerCamelCase     :      Optional[Any]       						=						1

            if year < 2_001 and day == 1:
                  sundays += 1
      return sundays


if __name__ == "__main__":
    print(solution())



