



"""simple docstring"""
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCAmelCase     =			re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

_UpperCAmelCase     =			None





def      __magic_name__				(							):
   SCREAMING_SNAKE_CASE_: Dict     			=argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
   parser.add_argument("""data_file"""							,			metavar="""data.json"""							,			help="""Input data JSON file.""" )
   parser.add_argument("""pred_file"""							,			metavar="""pred.json"""							,			help="""Model predictions.""" )
   parser.add_argument(
       """--out-file"""							,			"""-o"""							,			metavar="""eval.json"""							,			help="""Write accuracy metrics to file (default is stdout).""" )
   parser.add_argument(
       """--na-prob-file"""							,			"""-n"""							,			metavar="""na_prob.json"""							,			help="""Model estimates of probability of no answer.""" )
   parser.add_argument(
       """--na-prob-thresh"""							,			"""-t"""							,			type=lowercase							,			default=1.0							,			help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""							,			)
   parser.add_argument(
       """--out-image-dir"""							,			"""-p"""							,			metavar="""out_images"""							,			default=lowercase							,			help="""Save precision-recall curves to directory.""" )
   parser.add_argument("""--verbose"""							,			"""-v"""							,			action="""store_true""" )
   if len(sys.argv ) == 1:
      parser.print_help()
      sys.exit(1 )
   return parser.parse_args()





def      __magic_name__				(							lowercase ):
   SCREAMING_SNAKE_CASE_: str     			={}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            SCREAMING_SNAKE_CASE_: Dict     			=bool(qa["""answers"""]["""text"""] )
   return qid_to_has_ans





def      __magic_name__				(							lowercase ):

   def remove_articles(lowercase ):
      return ARTICLES_REGEX.sub(""" """							,			lowercase )

   def white_space_fix(lowercase ):
      return " ".join(text.split() )

   def remove_punc(lowercase ):
      SCREAMING_SNAKE_CASE_: Optional[Any]     			=set(string.punctuation )
      return "".join(ch for ch in text if ch not in exclude )

   def lower(lowercase ):
      return text.lower()

   return white_space_fix(remove_articles(remove_punc(lower(lowercase ) ) ) )





def      __magic_name__				(							lowercase ):
   if not s:
      return []
   return normalize_answer(lowercase ).split()





def      __magic_name__				(							lowercase							,			lowercase ):
   return int(normalize_answer(lowercase ) == normalize_answer(lowercase ) )





def      __magic_name__				(							lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_: List[Any]     			=get_tokens(lowercase )
   SCREAMING_SNAKE_CASE_: str     			=get_tokens(lowercase )
   SCREAMING_SNAKE_CASE_: Tuple     			=collections.Counter(lowercase ) & collections.Counter(lowercase )
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=sum(common.values() )
   if len(lowercase ) == 0 or len(lowercase ) == 0:
      # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
      return int(gold_toks == pred_toks )
   if num_same == 0:
      return 0
   SCREAMING_SNAKE_CASE_: Optional[int]     			=1.0 * num_same / len(lowercase )
   SCREAMING_SNAKE_CASE_: Any     			=1.0 * num_same / len(lowercase )
   SCREAMING_SNAKE_CASE_: str     			=(2 * precision * recall) / (precision + recall)
   return fa





def      __magic_name__				(							lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_: Dict     			={}
   SCREAMING_SNAKE_CASE_: List[Any]     			={}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            SCREAMING_SNAKE_CASE_: Union[str, Any]     			=qa["""id"""]
            SCREAMING_SNAKE_CASE_: Tuple     			=[t for t in qa["""answers"""]["""text"""] if normalize_answer(lowercase )]
            if not gold_answers:
               # For unanswerable questions, only correct answer is empty string
               SCREAMING_SNAKE_CASE_: Optional[int]     			=[""""""]
            if qid not in preds:
               print(f'''Missing prediction for {qid}''' )
               continue
            SCREAMING_SNAKE_CASE_: Dict     			=preds[qid]
            # Take max over all gold answers
            SCREAMING_SNAKE_CASE_: Tuple     			=max(compute_exact(lowercase							,			lowercase ) for a in gold_answers )
            SCREAMING_SNAKE_CASE_: Tuple     			=max(compute_fa(lowercase							,			lowercase ) for a in gold_answers )
   return exact_scores, fa_scores





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_: Optional[Any]     			={}
   for qid, s in scores.items():
      SCREAMING_SNAKE_CASE_: str     			=na_probs[qid] > na_prob_thresh
      if pred_na:
         SCREAMING_SNAKE_CASE_: Tuple     			=float(not qid_to_has_ans[qid] )
      else:
         SCREAMING_SNAKE_CASE_: Optional[int]     			=s
   return new_scores





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase=None ):
   if not qid_list:
      SCREAMING_SNAKE_CASE_: Dict     			=len(lowercase )
      return collections.OrderedDict(
          [
              ("""exact""", 100.0 * sum(exact_scores.values() ) / total),
              ("""f1""", 100.0 * sum(fa_scores.values() ) / total),
              ("""total""", total),
          ] )
   else:
      SCREAMING_SNAKE_CASE_: Dict     			=len(lowercase )
      return collections.OrderedDict(
          [
              ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list ) / total),
              ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list ) / total),
              ("""total""", total),
          ] )





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase ):
   for k in new_eval:
      SCREAMING_SNAKE_CASE_: Tuple     			=new_eval[k]





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase ):
   plt.step(lowercase							,			lowercase							,			color="""b"""							,			alpha=0.2							,			where="""post""" )
   plt.fill_between(lowercase							,			lowercase							,			step="""post"""							,			alpha=0.2							,			color="""b""" )
   plt.xlabel("""Recall""" )
   plt.ylabel("""Precision""" )
   plt.xlim([0.0, 1.05] )
   plt.ylim([0.0, 1.05] )
   plt.title(lowercase )
   plt.savefig(lowercase )
   plt.clf()





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase=None							,			lowercase=None ):
   SCREAMING_SNAKE_CASE_: Union[str, Any]     			=sorted(lowercase							,			key=lambda lowercase : na_probs[k] )
   SCREAMING_SNAKE_CASE_: List[Any]     			=0.0
   SCREAMING_SNAKE_CASE_: Dict     			=1.0
   SCREAMING_SNAKE_CASE_: Tuple     			=0.0
   SCREAMING_SNAKE_CASE_: Any     			=[1.0]
   SCREAMING_SNAKE_CASE_: str     			=[0.0]
   SCREAMING_SNAKE_CASE_: Optional[int]     			=0.0
   for i, qid in enumerate(lowercase ):
      if qid_to_has_ans[qid]:
         true_pos += scores[qid]
      SCREAMING_SNAKE_CASE_: List[Any]     			=true_pos / float(i + 1 )
      SCREAMING_SNAKE_CASE_: List[Any]     			=true_pos / float(lowercase )
      if i == len(lowercase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
         # i.e., if we can put a threshold after this point
         avg_prec += cur_p * (cur_r - recalls[-1])
         precisions.append(lowercase )
         recalls.append(lowercase )
   if out_image:
      plot_pr_curve(lowercase							,			lowercase							,			lowercase							,			lowercase )
   return {"ap": 100.0 * avg_prec}





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase ):
   if out_image_dir and not os.path.exists(lowercase ):
      os.makedirs(lowercase )
   SCREAMING_SNAKE_CASE_: Dict     			=sum(1 for v in qid_to_has_ans.values() if v )
   if num_true_pos == 0:
      return
   SCREAMING_SNAKE_CASE_: Optional[int]     			=make_precision_recall_eval(
       lowercase							,			lowercase							,			lowercase							,			lowercase							,			out_image=os.path.join(lowercase							,			"""pr_exact.png""" )							,			title="""Precision-Recall curve for Exact Match score"""							,			)
   SCREAMING_SNAKE_CASE_: Dict     			=make_precision_recall_eval(
       lowercase							,			lowercase							,			lowercase							,			lowercase							,			out_image=os.path.join(lowercase							,			"""pr_f1.png""" )							,			title="""Precision-Recall curve for F1 score"""							,			)
   SCREAMING_SNAKE_CASE_: Any     			={k: float(lowercase ) for k, v in qid_to_has_ans.items()}
   SCREAMING_SNAKE_CASE_: str     			=make_precision_recall_eval(
       lowercase							,			lowercase							,			lowercase							,			lowercase							,			out_image=os.path.join(lowercase							,			"""pr_oracle.png""" )							,			title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""							,			)
   merge_eval(lowercase							,			lowercase							,			"""pr_exact""" )
   merge_eval(lowercase							,			lowercase							,			"""pr_f1""" )
   merge_eval(lowercase							,			lowercase							,			"""pr_oracle""" )





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase ):
   if not qid_list:
      return
   SCREAMING_SNAKE_CASE_: Union[str, Any]     			=[na_probs[k] for k in qid_list]
   SCREAMING_SNAKE_CASE_: Any     			=np.ones_like(lowercase ) / float(len(lowercase ) )
   plt.hist(lowercase							,			weights=lowercase							,			bins=20							,			range=(0.0, 1.0) )
   plt.xlabel("""Model probability of no-answer""" )
   plt.ylabel("""Proportion of dataset""" )
   plt.title(f'''Histogram of no-answer probability: {name}''' )
   plt.savefig(os.path.join(lowercase							,			f'''na_prob_hist_{name}.png''' ) )
   plt.clf()





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_: int     			=sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
   SCREAMING_SNAKE_CASE_: str     			=num_no_ans
   SCREAMING_SNAKE_CASE_: Dict     			=cur_score
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=0.0
   SCREAMING_SNAKE_CASE_: Dict     			=sorted(lowercase							,			key=lambda lowercase : na_probs[k] )
   for i, qid in enumerate(lowercase ):
      if qid not in scores:
         continue
      if qid_to_has_ans[qid]:
         SCREAMING_SNAKE_CASE_: Dict     			=scores[qid]
      else:
         if preds[qid]:
            SCREAMING_SNAKE_CASE_: Union[str, Any]     			=-1
         else:
            SCREAMING_SNAKE_CASE_: Tuple     			=0
      cur_score += diff
      if cur_score > best_score:
         SCREAMING_SNAKE_CASE_: int     			=cur_score
         SCREAMING_SNAKE_CASE_: int     			=na_probs[qid]
   return 100.0 * best_score / len(lowercase ), best_thresh





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: List[str]     			=find_best_thresh(lowercase							,			lowercase							,			lowercase							,			lowercase )
   SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: List[str]     			=find_best_thresh(lowercase							,			lowercase							,			lowercase							,			lowercase )
   SCREAMING_SNAKE_CASE_: int     			=best_exact
   SCREAMING_SNAKE_CASE_: Any     			=exact_thresh
   SCREAMING_SNAKE_CASE_: str     			=best_fa
   SCREAMING_SNAKE_CASE_: Any     			=fa_thresh





def      __magic_name__				(							):
   with open(OPTS.data_file ) as f:
      SCREAMING_SNAKE_CASE_: Any     			=json.load(lowercase )
      SCREAMING_SNAKE_CASE_: Optional[Any]     			=dataset_json["""data"""]
   with open(OPTS.pred_file ) as f:
      SCREAMING_SNAKE_CASE_: List[Any]     			=json.load(lowercase )
   if OPTS.na_prob_file:
      with open(OPTS.na_prob_file ) as f:
         SCREAMING_SNAKE_CASE_: Union[str, Any]     			=json.load(lowercase )
   else:
      SCREAMING_SNAKE_CASE_: int     			={k: 0.0 for k in preds}
   SCREAMING_SNAKE_CASE_: List[str]     			=make_qid_to_has_ans(lowercase )  # maps qid to True/False
   SCREAMING_SNAKE_CASE_: str     			=[k for k, v in qid_to_has_ans.items() if v]
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=[k for k, v in qid_to_has_ans.items() if not v]
   SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: Union[str, Any]     			=get_raw_scores(lowercase							,			lowercase )
   SCREAMING_SNAKE_CASE_: Any     			=apply_no_ans_threshold(lowercase							,			lowercase							,			lowercase							,			OPTS.na_prob_thresh )
   SCREAMING_SNAKE_CASE_: Optional[int]     			=apply_no_ans_threshold(lowercase							,			lowercase							,			lowercase							,			OPTS.na_prob_thresh )
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=make_eval_dict(lowercase							,			lowercase )
   if has_ans_qids:
      SCREAMING_SNAKE_CASE_: int     			=make_eval_dict(lowercase							,			lowercase							,			qid_list=lowercase )
      merge_eval(lowercase							,			lowercase							,			"""HasAns""" )
   if no_ans_qids:
      SCREAMING_SNAKE_CASE_: Union[str, Any]     			=make_eval_dict(lowercase							,			lowercase							,			qid_list=lowercase )
      merge_eval(lowercase							,			lowercase							,			"""NoAns""" )
   if OPTS.na_prob_file:
      find_all_best_thresh(lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase )
   if OPTS.na_prob_file and OPTS.out_image_dir:
      run_precision_recall_analysis(lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase							,			OPTS.out_image_dir )
      histogram_na_prob(lowercase							,			lowercase							,			OPTS.out_image_dir							,			"""hasAns""" )
      histogram_na_prob(lowercase							,			lowercase							,			OPTS.out_image_dir							,			"""noAns""" )
   if OPTS.out_file:
      with open(OPTS.out_file							,			"""w""" ) as f:
         json.dump(lowercase							,			lowercase )
   else:
      print(json.dumps(lowercase							,			indent=2 ) )


if __name__ == "__main__":
   _UpperCAmelCase     =			parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use("""Agg""")
      import matplotlib.pyplot as plt
   main()








"""simple docstring"""
import string





def      __magic_name__				(							lowercase ):
   SCREAMING_SNAKE_CASE_: List[Any]     			=""""""
   for i in sequence:
      SCREAMING_SNAKE_CASE_: List[Any]     			=ord(lowercase )
      if 65 <= extract <= 90:
         output += chr(155 - extract )
      elif 97 <= extract <= 122:
         output += chr(219 - extract )
      else:
         output += i
   return output





def      __magic_name__				(							lowercase ):
   SCREAMING_SNAKE_CASE_: Any     			=string.ascii_letters
   SCREAMING_SNAKE_CASE_: Tuple     			=string.ascii_lowercase[::-1] + string.ascii_uppercase[::-1]
   return "".join(
       letters_reversed[letters.index(lowercase )] if c in letters else c for c in sequence )





def      __magic_name__				(							):
   from timeit import timeit

   print("""Running performance benchmarks...""" )
   SCREAMING_SNAKE_CASE_: int     			="""from string import printable ; from __main__ import atbash, atbash_slow"""
   print(f'''> atbash_slow(): {timeit("atbash_slow(printable)"							,			setup=lowercase )} seconds''' )
   print(f'''>      atbash(): {timeit("atbash(printable)"							,			setup=lowercase )} seconds''' )


if __name__ == "__main__":
   for example in ("ABCDEFGH", "123GGjj", "testStringtest", "with space"):
      print(f"""{example} encrypted in atbash: {atbash(example)}""")
   benchmark()



