


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase        =     re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__lowerCamelCase        =     None



def  _a			(						):
     a_    :     Tuple										=    argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
     parser.add_argument("""data_file"""  ,  metavar="""data.json"""  ,  help="""Input data JSON file.""" )
     parser.add_argument("""pred_file"""  ,  metavar="""pred.json"""  ,  help="""Model predictions.""" )
     parser.add_argument(
         """--out-file"""  ,  """-o"""  ,  metavar="""eval.json"""  ,  help="""Write accuracy metrics to file (default is stdout).""" )
     parser.add_argument(
         """--na-prob-file"""  ,  """-n"""  ,  metavar="""na_prob.json"""  ,  help="""Model estimates of probability of no answer.""" )
     parser.add_argument(
         """--na-prob-thresh"""  ,  """-t"""  ,  type=__UpperCamelCase  ,  default=1.0  ,  help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""  ,  )
     parser.add_argument(
         """--out-image-dir"""  ,  """-p"""  ,  metavar="""out_images"""  ,  default=__UpperCamelCase  ,  help="""Save precision-recall curves to directory.""" )
     parser.add_argument("""--verbose"""  ,  """-v"""  ,  action="""store_true""" )
     if len(sys.argv ) == 1:
          parser.print_help()
          sys.exit(1 )
     return parser.parse_args()



def  _a			(						__UpperCamelCase ):
     a_    :     Union[str, Any]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     Union[str, Any]										=    bool(qa["""answers"""]["""text"""] )
     return qid_to_has_ans



def  _a			(						__UpperCamelCase ):

     def remove_articles(__UpperCamelCase ):
          return ARTICLES_REGEX.sub(""" """  ,  __UpperCamelCase )

     def white_space_fix(__UpperCamelCase ):
          return " ".join(text.split() )

     def remove_punc(__UpperCamelCase ):
          a_    :     List[str]										=    set(string.punctuation )
          return "".join(ch for ch in text if ch not in exclude )

     def lower(__UpperCamelCase ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase ) ) ) )



def  _a			(						__UpperCamelCase ):
     if not s:
          return []
     return normalize_answer(__UpperCamelCase ).split()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     return int(normalize_answer(__UpperCamelCase ) == normalize_answer(__UpperCamelCase ) )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     List[Any]										=    get_tokens(__UpperCamelCase )
     a_    :     str										=    get_tokens(__UpperCamelCase )
     a_    :     Tuple										=    collections.Counter(__UpperCamelCase ) & collections.Counter(__UpperCamelCase )
     a_    :     str										=    sum(common.values() )
     if len(__UpperCamelCase ) == 0 or len(__UpperCamelCase ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks )
     if num_same == 0:
          return 0
     a_    :     Any										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     Optional[int]										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     List[Any]										=    (2 * precision * recall) / (precision + recall)
     return fa



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Tuple										=    {}
     a_    :     Optional[int]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     str										=    qa["""id"""]
                    a_    :     Dict										=    [t for t in qa["""answers"""]["""text"""] if normalize_answer(__UpperCamelCase )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         a_    :     Union[str, Any]										=    [""""""]
                    if qid not in preds:
                         print(F'''Missing prediction for {qid}''' )
                         continue
                    a_    :     int										=    preds[qid]
                    # Take max over all gold answers
                    a_    :     Union[str, Any]										=    max(compute_exact(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
                    a_    :     int										=    max(compute_fa(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
     return exact_scores, fa_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[Any]										=    {}
     for qid, s in scores.items():
          a_    :     Optional[int]										=    na_probs[qid] > na_prob_thresh
          if pred_na:
               a_    :     List[Any]										=    float(not qid_to_has_ans[qid] )
          else:
               a_    :     List[Any]										=    s
     return new_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None ):
     if not qid_list:
          a_    :     List[Any]										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores.values() ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores.values() ) / total),
                  ("""total""", total),
              ] )
     else:
          a_    :     Tuple										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores[k] for k in qid_list ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores[k] for k in qid_list ) / total),
                  ("""total""", total),
              ] )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     for k in new_eval:
          a_    :     Any										=    new_eval[k]



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     plt.step(__UpperCamelCase  ,  __UpperCamelCase  ,  color="""b"""  ,  alpha=0.2  ,  where="""post""" )
     plt.fill_between(__UpperCamelCase  ,  __UpperCamelCase  ,  step="""post"""  ,  alpha=0.2  ,  color="""b""" )
     plt.xlabel("""Recall""" )
     plt.ylabel("""Precision""" )
     plt.xlim([0.0, 1.05] )
     plt.ylim([0.0, 1.05] )
     plt.title(__UpperCamelCase )
     plt.savefig(__UpperCamelCase )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None  ,  __UpperCamelCase=None ):
     a_    :     List[Any]										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     a_    :     int										=    0.0
     a_    :     Tuple										=    1.0
     a_    :     Union[str, Any]										=    0.0
     a_    :     List[str]										=    [1.0]
     a_    :     int										=    [0.0]
     a_    :     Optional[int]										=    0.0
     for i, qid in enumerate(__UpperCamelCase ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          a_    :     Union[str, Any]										=    true_pos / float(i + 1 )
          a_    :     Tuple										=    true_pos / float(__UpperCamelCase )
          if i == len(__UpperCamelCase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(__UpperCamelCase )
               recalls.append(__UpperCamelCase )
     if out_image:
          plot_pr_curve(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     return {"ap": 1_00.0 * avg_prec}



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if out_image_dir and not os.path.exists(__UpperCamelCase ):
          os.makedirs(__UpperCamelCase )
     a_    :     Any										=    sum(1 for v in qid_to_has_ans.values() if v )
     if num_true_pos == 0:
          return
     a_    :     Union[str, Any]										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_exact.png""" )  ,  title="""Precision-Recall curve for Exact Match score"""  ,  )
     a_    :     Any										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_f1.png""" )  ,  title="""Precision-Recall curve for F1 score"""  ,  )
     a_    :     int										=    {k: float(__UpperCamelCase ) for k, v in qid_to_has_ans.items()}
     a_    :     Dict										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_oracle.png""" )  ,  title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""  ,  )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_exact""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_f1""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_oracle""" )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if not qid_list:
          return
     a_    :     List[Any]										=    [na_probs[k] for k in qid_list]
     a_    :     List[Any]										=    np.ones_like(__UpperCamelCase ) / float(len(__UpperCamelCase ) )
     plt.hist(__UpperCamelCase  ,  weights=__UpperCamelCase  ,  bins=2_0  ,  range=(0.0, 1.0) )
     plt.xlabel("""Model probability of no-answer""" )
     plt.ylabel("""Proportion of dataset""" )
     plt.title(F'''Histogram of no-answer probability: {name}''' )
     plt.savefig(os.path.join(__UpperCamelCase  ,  F'''na_prob_hist_{name}.png''' ) )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[int]										=    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
     a_    :     Tuple										=    num_no_ans
     a_    :     List[Any]										=    cur_score
     a_    :     List[Any]										=    0.0
     a_    :     int										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     for i, qid in enumerate(__UpperCamelCase ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               a_    :     Optional[Any]										=    scores[qid]
          else:
               if preds[qid]:
                    a_    :     Optional[int]										=    -1
               else:
                    a_    :     Optional[int]										=    0
          cur_score += diff
          if cur_score > best_score:
               a_    :     str										=    cur_score
               a_    :     int										=    na_probs[qid]
     return 1_00.0 * best_score / len(__UpperCamelCase ), best_thresh



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_			,			a_    :     Optional[int]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_			,			a_    :     Optional[Any]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_    :     Dict										=    best_exact
     a_    :     Optional[Any]										=    exact_thresh
     a_    :     Optional[Any]										=    best_fa
     a_    :     Union[str, Any]										=    fa_thresh



def  _a			(						):
     with open(OPTS.data_file ) as f:
          a_    :     str										=    json.load(__UpperCamelCase )
          a_    :     Optional[int]										=    dataset_json["""data"""]
     with open(OPTS.pred_file ) as f:
          a_    :     Dict										=    json.load(__UpperCamelCase )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file ) as f:
               a_    :     List[str]										=    json.load(__UpperCamelCase )
     else:
          a_    :     List[str]										=    {k: 0.0 for k in preds}
     a_    :     Any										=    make_qid_to_has_ans(__UpperCamelCase )  # maps qid to True/False
     a_    :     Any										=    [k for k, v in qid_to_has_ans.items() if v]
     a_    :     Union[str, Any]										=    [k for k, v in qid_to_has_ans.items() if not v]
     a_			,			a_    :     List[str]										=    get_raw_scores(__UpperCamelCase  ,  __UpperCamelCase )
     a_    :     int										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     Optional[int]										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase )
     if has_ans_qids:
          a_    :     Dict										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """HasAns""" )
     if no_ans_qids:
          a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """NoAns""" )
     if OPTS.na_prob_file:
          find_all_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """hasAns""" )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """noAns""" )
     if OPTS.out_file:
          with open(OPTS.out_file  ,  """w""" ) as f:
               json.dump(__UpperCamelCase  ,  __UpperCamelCase )
     else:
          print(json.dumps(__UpperCamelCase  ,  indent=2 ) )


if __name__ == "__main__":
       __lowerCamelCase        =     parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('''Agg''')
              import matplotlib.pyplot as plt
       main()







from heapq import heappop, heappush

import numpy as np



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  ):
     a_			,			a_    :     Any										=    grid.shape
     a_    :     Dict										=    [-1, 1, 0, 0]
     a_    :     List[Any]										=    [0, 0, -1, 1]
     if allow_diagonal:
          dx += [-1, -1, 1, 1]
          dy += [-1, 1, -1, 1]

     a_			,			a_    :     int										=    [(0, source)], set()
     a_    :     Optional[Any]										=    np.full((rows, cols)  ,  np.inf )
     a_    :     Tuple										=    0
     a_    :     Optional[Any]										=    np.empty((rows, cols)  ,  dtype=__UpperCamelCase )
     a_    :     Optional[Any]										=    None

     while queue:
          ((a_)			,			(a_))    :     Any										=    heappop(__UpperCamelCase )
          if (x, y) in visited:
               continue
          visited.add((x, y) )

          if (x, y) == destination:
               a_    :     Tuple										=    []
               while (x, y) != source:
                    path.append((x, y) )
                    a_			,			a_    :     Any										=    predecessors[x, y]
               path.append(__UpperCamelCase )  # add the source manually
               path.reverse()
               return matrix[destination], path

          for i in range(len(__UpperCamelCase ) ):
               a_			,			a_    :     Dict										=    x + dx[i], y + dy[i]
               if 0 <= nx < rows and 0 <= ny < cols:
                    a_    :     List[Any]										=    grid[nx][ny]
                    if next_node == 1 and matrix[nx, ny] > dist + 1:
                         heappush(__UpperCamelCase  ,  (dist + 1, (nx, ny)) )
                         a_    :     List[str]										=    dist + 1
                         a_    :     Union[str, Any]										=    (x, y)

     return np.inf, []


if __name__ == "__main__":
       import doctest

       doctest.testmod()



