
'''simple docstring'''
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


A__   :		Optional[int] 		=re.compile(r'\b(a|an|the)\b', re.UNICODE)

A__   :		List[str] 		=None







def        A_     (						)				->       Optional[Any]:





  """simple docstring"""

  __A		:     int												=		argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
  parser.add_argument("""data_file"""			,     metavar="""data.json"""			,     help="""Input data JSON file.""" )
  parser.add_argument("""pred_file"""			,     metavar="""pred.json"""			,     help="""Model predictions.""" )
  parser.add_argument(
      """--out-file"""			,     """-o"""			,     metavar="""eval.json"""			,     help="""Write accuracy metrics to file (default is stdout).""" )
  parser.add_argument(
      """--na-prob-file"""			,     """-n"""			,     metavar="""na_prob.json"""			,     help="""Model estimates of probability of no answer.""" )
  parser.add_argument(
      """--na-prob-thresh"""			,     """-t"""			,     type=__lowerCAmelCase			,     default=1.0			,     help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""			,     )
  parser.add_argument(
      """--out-image-dir"""			,     """-p"""			,     metavar="""out_images"""			,     default=__lowerCAmelCase			,     help="""Save precision-recall curves to directory.""" )
  parser.add_argument("""--verbose"""			,     """-v"""			,     action="""store_true""" )
  if len(sys.argv ) == 1:
    parser.print_help()
    sys.exit(1 )
  return parser.parse_args()







def        A_     (						__SCREAMING_SNAKE_CASE						:  int )				->       Optional[Any]:





  """simple docstring"""

  __A		:     Optional[int]												=		{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        __A		:     Optional[int]												=		bool(qa["""answers"""]["""text"""] )
  return qid_to_has_ans







def        A_     (						__SCREAMING_SNAKE_CASE						:  Union[str, Any] )				->       str:





  """simple docstring"""

  def remove_articles(__SCREAMING_SNAKE_CASE						:  int ):
    return ARTICLES_REGEX.sub(""" """			,     __lowerCAmelCase )

  def white_space_fix(__SCREAMING_SNAKE_CASE						:  List[str] ):
    return " ".join(text.split() )

  def remove_punc(__SCREAMING_SNAKE_CASE						:  List[str] ):
    __A		:     Tuple												=		set(string.punctuation )
    return "".join(ch for ch in text if ch not in exclude )

  def lower(__SCREAMING_SNAKE_CASE						:  Optional[Any] ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase ) ) ) )







def        A_     (						__SCREAMING_SNAKE_CASE						:  List[str] )				->       Any:





  """simple docstring"""

  if not s:
    return []
  return normalize_answer(__lowerCAmelCase ).split()







def        A_     (						__SCREAMING_SNAKE_CASE						:  List[str]			,     __SCREAMING_SNAKE_CASE						:  Optional[int] )				->       Dict:





  """simple docstring"""

  return int(normalize_answer(__lowerCAmelCase ) == normalize_answer(__lowerCAmelCase ) )







def        A_     (						__SCREAMING_SNAKE_CASE						:  Optional[Any]			,     __SCREAMING_SNAKE_CASE						:  Tuple )				->       Union[str, Any]:





  """simple docstring"""

  __A		:     List[Any]												=		get_tokens(__lowerCAmelCase )
  __A		:     List[Any]												=		get_tokens(__lowerCAmelCase )
  __A		:     List[Any]												=		collections.Counter(__lowerCAmelCase ) & collections.Counter(__lowerCAmelCase )
  __A		:     List[str]												=		sum(common.values() )
  if len(__lowerCAmelCase ) == 0 or len(__lowerCAmelCase ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks )
  if num_same == 0:
    return 0
  __A		:     Union[str, Any]												=		1.0 * num_same / len(__lowerCAmelCase )
  __A		:     Tuple												=		1.0 * num_same / len(__lowerCAmelCase )
  __A		:     Union[str, Any]												=		(2 * precision * recall) / (precision + recall)
  return fa







def        A_     (						__SCREAMING_SNAKE_CASE						:  int			,     __SCREAMING_SNAKE_CASE						:  Dict )				->       Dict:





  """simple docstring"""

  __A		:     Any												=		{}
  __A		:     List[Any]												=		{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        __A		:     Optional[Any]												=		qa["""id"""]
        __A		:     Optional[Any]												=		[t for t in qa["""answers"""]["""text"""] if normalize_answer(__lowerCAmelCase )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          __A		:     List[Any]												=		[""""""]
        if qid not in preds:
          print(F"Missing prediction for {qid}" )
          continue
        __A		:     List[Any]												=		preds[qid]
        # Take max over all gold answers
        __A		:     Any												=		max(compute_exact(__lowerCAmelCase			,     __lowerCAmelCase ) for a in gold_answers )
        __A		:     str												=		max(compute_fa(__lowerCAmelCase			,     __lowerCAmelCase ) for a in gold_answers )
  return exact_scores, fa_scores







def        A_     (						__SCREAMING_SNAKE_CASE						:  int			,     __SCREAMING_SNAKE_CASE						:  Dict			,     __SCREAMING_SNAKE_CASE						:  str			,     __SCREAMING_SNAKE_CASE						:  Union[str, Any] )				->       int:





  """simple docstring"""

  __A		:     str												=		{}
  for qid, s in scores.items():
    __A		:     Any												=		na_probs[qid] > na_prob_thresh
    if pred_na:
      __A		:     List[str]												=		float(not qid_to_has_ans[qid] )
    else:
      __A		:     Optional[Any]												=		s
  return new_scores







def        A_     (						__SCREAMING_SNAKE_CASE						:  int			,     __SCREAMING_SNAKE_CASE						:  str			,     __SCREAMING_SNAKE_CASE						:  int=None )				->       Union[str, Any]:





  """simple docstring"""

  if not qid_list:
    __A		:     int												=		len(__lowerCAmelCase )
    return collections.OrderedDict(
        [
            ("""exact""", 100.0 * sum(exact_scores.values() ) / total),
            ("""f1""", 100.0 * sum(fa_scores.values() ) / total),
            ("""total""", total),
        ] )
  else:
    __A		:     str												=		len(__lowerCAmelCase )
    return collections.OrderedDict(
        [
            ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list ) / total),
            ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list ) / total),
            ("""total""", total),
        ] )







def        A_     (						__SCREAMING_SNAKE_CASE						:  Tuple			,     __SCREAMING_SNAKE_CASE						:  str			,     __SCREAMING_SNAKE_CASE						:  Union[str, Any] )				->       Dict:





  """simple docstring"""

  for k in new_eval:
    __A		:     Optional[int]												=		new_eval[k]







def        A_     (						__SCREAMING_SNAKE_CASE						:  Optional[int]			,     __SCREAMING_SNAKE_CASE						:  Optional[int]			,     __SCREAMING_SNAKE_CASE						:  List[Any]			,     __SCREAMING_SNAKE_CASE						:  List[Any] )				->       Dict:





  """simple docstring"""

  plt.step(__lowerCAmelCase			,     __lowerCAmelCase			,     color="""b"""			,     alpha=0.2			,     where="""post""" )
  plt.fill_between(__lowerCAmelCase			,     __lowerCAmelCase			,     step="""post"""			,     alpha=0.2			,     color="""b""" )
  plt.xlabel("""Recall""" )
  plt.ylabel("""Precision""" )
  plt.xlim([0.0, 1.05] )
  plt.ylim([0.0, 1.05] )
  plt.title(__lowerCAmelCase )
  plt.savefig(__lowerCAmelCase )
  plt.clf()







def        A_     (						__SCREAMING_SNAKE_CASE						:  Dict			,     __SCREAMING_SNAKE_CASE						:  Optional[Any]			,     __SCREAMING_SNAKE_CASE						:  List[str]			,     __SCREAMING_SNAKE_CASE						:  Optional[int]			,     __SCREAMING_SNAKE_CASE						:  List[Any]=None			,     __SCREAMING_SNAKE_CASE						:  Any=None )				->       str:





  """simple docstring"""

  __A		:     Union[str, Any]												=		sorted(__lowerCAmelCase			,     key=lambda __SCREAMING_SNAKE_CASE : na_probs[k] )
  __A		:     List[Any]												=		0.0
  __A		:     List[Any]												=		1.0
  __A		:     Union[str, Any]												=		0.0
  __A		:     List[str]												=		[1.0]
  __A		:     List[str]												=		[0.0]
  __A		:     Union[str, Any]												=		0.0
  for i, qid in enumerate(__lowerCAmelCase ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    __A		:     Optional[Any]												=		true_pos / float(i + 1 )
    __A		:     Optional[Any]												=		true_pos / float(__lowerCAmelCase )
    if i == len(__lowerCAmelCase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(__lowerCAmelCase )
      recalls.append(__lowerCAmelCase )
  if out_image:
    plot_pr_curve(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase )
  return {"ap": 100.0 * avg_prec}







def        A_     (						__SCREAMING_SNAKE_CASE						:  Optional[Any]			,     __SCREAMING_SNAKE_CASE						:  int			,     __SCREAMING_SNAKE_CASE						:  Union[str, Any]			,     __SCREAMING_SNAKE_CASE						:  List[str]			,     __SCREAMING_SNAKE_CASE						:  List[Any]			,     __SCREAMING_SNAKE_CASE						:  List[str] )				->       Optional[Any]:





  """simple docstring"""

  if out_image_dir and not os.path.exists(__lowerCAmelCase ):
    os.makedirs(__lowerCAmelCase )
  __A		:     List[Any]												=		sum(1 for v in qid_to_has_ans.values() if v )
  if num_true_pos == 0:
    return
  __A		:     Union[str, Any]												=		make_precision_recall_eval(
      __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     out_image=os.path.join(__lowerCAmelCase			,     """pr_exact.png""" )			,     title="""Precision-Recall curve for Exact Match score"""			,     )
  __A		:     Tuple												=		make_precision_recall_eval(
      __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     out_image=os.path.join(__lowerCAmelCase			,     """pr_f1.png""" )			,     title="""Precision-Recall curve for F1 score"""			,     )
  __A		:     Any												=		{k: float(__lowerCAmelCase ) for k, v in qid_to_has_ans.items()}
  __A		:     Tuple												=		make_precision_recall_eval(
      __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     out_image=os.path.join(__lowerCAmelCase			,     """pr_oracle.png""" )			,     title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""			,     )
  merge_eval(__lowerCAmelCase			,     __lowerCAmelCase			,     """pr_exact""" )
  merge_eval(__lowerCAmelCase			,     __lowerCAmelCase			,     """pr_f1""" )
  merge_eval(__lowerCAmelCase			,     __lowerCAmelCase			,     """pr_oracle""" )







def        A_     (						__SCREAMING_SNAKE_CASE						:  str			,     __SCREAMING_SNAKE_CASE						:  List[str]			,     __SCREAMING_SNAKE_CASE						:  List[Any]			,     __SCREAMING_SNAKE_CASE						:  Optional[int] )				->       List[str]:





  """simple docstring"""

  if not qid_list:
    return
  __A		:     Optional[Any]												=		[na_probs[k] for k in qid_list]
  __A		:     Any												=		np.ones_like(__lowerCAmelCase ) / float(len(__lowerCAmelCase ) )
  plt.hist(__lowerCAmelCase			,     weights=__lowerCAmelCase			,     bins=20			,     range=(0.0, 1.0) )
  plt.xlabel("""Model probability of no-answer""" )
  plt.ylabel("""Proportion of dataset""" )
  plt.title(F"Histogram of no-answer probability: {name}" )
  plt.savefig(os.path.join(__lowerCAmelCase			,     F"na_prob_hist_{name}.png" ) )
  plt.clf()







def        A_     (						__SCREAMING_SNAKE_CASE						:  List[Any]			,     __SCREAMING_SNAKE_CASE						:  Optional[int]			,     __SCREAMING_SNAKE_CASE						:  Union[str, Any]			,     __SCREAMING_SNAKE_CASE						:  Tuple )				->       str:





  """simple docstring"""

  __A		:     Optional[int]												=		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
  __A		:     Any												=		num_no_ans
  __A		:     int												=		cur_score
  __A		:     List[Any]												=		0.0
  __A		:     Any												=		sorted(__lowerCAmelCase			,     key=lambda __SCREAMING_SNAKE_CASE : na_probs[k] )
  for i, qid in enumerate(__lowerCAmelCase ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      __A		:     Any												=		scores[qid]
    else:
      if preds[qid]:
        __A		:     Optional[Any]												=		-1
      else:
        __A		:     Tuple												=		0
    cur_score += diff
    if cur_score > best_score:
      __A		:     List[str]												=		cur_score
      __A		:     Dict												=		na_probs[qid]
  return 100.0 * best_score / len(__lowerCAmelCase ), best_thresh







def        A_     (						__SCREAMING_SNAKE_CASE						:  Union[str, Any]			,     __SCREAMING_SNAKE_CASE						:  Optional[Any]			,     __SCREAMING_SNAKE_CASE						:  Optional[Any]			,     __SCREAMING_SNAKE_CASE						:  Union[str, Any]			,     __SCREAMING_SNAKE_CASE						:  List[Any]			,     __SCREAMING_SNAKE_CASE						:  Dict )				->       int:





  """simple docstring"""

  __A		:     Dict												=		find_best_thresh(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase )
  __A		:     List[Any]												=		find_best_thresh(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase )
  __A		:     str												=		best_exact
  __A		:     Any												=		exact_thresh
  __A		:     Optional[Any]												=		best_fa
  __A		:     List[str]												=		fa_thresh







def        A_     (						)				->       List[Any]:





  """simple docstring"""

  with open(OPTS.data_file ) as f:
    __A		:     int												=		json.load(__lowerCAmelCase )
    __A		:     str												=		dataset_json["""data"""]
  with open(OPTS.pred_file ) as f:
    __A		:     Dict												=		json.load(__lowerCAmelCase )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file ) as f:
      __A		:     int												=		json.load(__lowerCAmelCase )
  else:
    __A		:     Tuple												=		{k: 0.0 for k in preds}
  __A		:     Union[str, Any]												=		make_qid_to_has_ans(__lowerCAmelCase )  # maps qid to True/False
  __A		:     Any												=		[k for k, v in qid_to_has_ans.items() if v]
  __A		:     int												=		[k for k, v in qid_to_has_ans.items() if not v]
  __A		:     Union[str, Any]												=		get_raw_scores(__lowerCAmelCase			,     __lowerCAmelCase )
  __A		:     Dict												=		apply_no_ans_threshold(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     OPTS.na_prob_thresh )
  __A		:     str												=		apply_no_ans_threshold(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     OPTS.na_prob_thresh )
  __A		:     List[Any]												=		make_eval_dict(__lowerCAmelCase			,     __lowerCAmelCase )
  if has_ans_qids:
    __A		:     int												=		make_eval_dict(__lowerCAmelCase			,     __lowerCAmelCase			,     qid_list=__lowerCAmelCase )
    merge_eval(__lowerCAmelCase			,     __lowerCAmelCase			,     """HasAns""" )
  if no_ans_qids:
    __A		:     Union[str, Any]												=		make_eval_dict(__lowerCAmelCase			,     __lowerCAmelCase			,     qid_list=__lowerCAmelCase )
    merge_eval(__lowerCAmelCase			,     __lowerCAmelCase			,     """NoAns""" )
  if OPTS.na_prob_file:
    find_all_best_thresh(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(__lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     __lowerCAmelCase			,     OPTS.out_image_dir )
    histogram_na_prob(__lowerCAmelCase			,     __lowerCAmelCase			,     OPTS.out_image_dir			,     """hasAns""" )
    histogram_na_prob(__lowerCAmelCase			,     __lowerCAmelCase			,     OPTS.out_image_dir			,     """noAns""" )
  if OPTS.out_file:
    with open(OPTS.out_file			,     """w""" ) as f:
      json.dump(__lowerCAmelCase			,     __lowerCAmelCase )
  else:
    print(json.dumps(__lowerCAmelCase			,     indent=2 ) )


if __name__ == "__main__":
     A__   :		Dict 		=parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use('Agg')
          import matplotlib.pyplot as plt
     main()





'''simple docstring'''
import argparse
import math
import traceback

import dateutil.parser as date_parser
import requests







def        A_     (						__SCREAMING_SNAKE_CASE						:  Dict )				->       Tuple:





  """simple docstring"""

  __A		:     Dict												=		{}

  __A		:     Optional[Any]												=		job["""started_at"""]
  __A		:     Tuple												=		job["""completed_at"""]

  __A		:     Optional[Any]												=		date_parser.parse(__SCREAMING_SNAKE_CASE )
  __A		:     List[Any]												=		date_parser.parse(__SCREAMING_SNAKE_CASE )

  __A		:     Union[str, Any]												=		round((end_datetime - start_datetime).total_seconds() / 60.0 )

  __A		:     List[Any]												=		start
  __A		:     Tuple												=		end
  __A		:     int												=		duration_in_min

  return job_info







def        A_     (						__SCREAMING_SNAKE_CASE						:  Union[str, Any]			,     __SCREAMING_SNAKE_CASE						:  int=None )				->       Dict:





  """simple docstring"""

  __A		:     List[Any]												=		None
  if token is not None:
    __A		:     Dict												=		{"""Accept""": """application/vnd.github+json""", """Authorization""": F"Bearer {token}"}

  __A		:     Tuple												=		F"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"
  __A		:     List[str]												=		requests.get(__SCREAMING_SNAKE_CASE			,     headers=__SCREAMING_SNAKE_CASE ).json()
  __A		:     List[str]												=		{}

  try:
    job_time.update({job["""name"""]: extract_time_from_single_job(__SCREAMING_SNAKE_CASE ) for job in result["""jobs"""]} )
    __A		:     Dict												=		math.ceil((result["""total_count"""] - 100) / 100 )

    for i in range(__SCREAMING_SNAKE_CASE ):
      __A		:     int												=		requests.get(url + F"&page={i + 2}"			,     headers=__SCREAMING_SNAKE_CASE ).json()
      job_time.update({job["""name"""]: extract_time_from_single_job(__SCREAMING_SNAKE_CASE ) for job in result["""jobs"""]} )

    return job_time
  except Exception:
    print(F"Unknown error, could not fetch links:\n{traceback.format_exc()}" )

  return {}


if __name__ == "__main__":
     A__   :		Any 		=argparse.ArgumentParser()
     # Required parameters
     parser.add_argument('--workflow_run_id', type=str, required=True, help='A GitHub Actions workflow run id.')
     A__   :		Any 		=parser.parse_args()

     A__   :		Any 		=get_job_time(args.workflow_run_id)
     A__   :		Dict 		=dict(sorted(job_time.items(), key=lambda item: item[1]["duration"], reverse=True))

     for k, v in job_time.items():
          print(F"{k}: {v['duration']}")



