





'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase					:		List[str]		=							re.compile(R'\b(a|an|the)\b', re.UNICODE)

UpperCAmelCase					:		Dict		=							None




def    _a    (							):





      """simple docstring"""



      _snake_case		:	int					     =     argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''	)
      parser.add_argument('''data_file'''			,      metavar='''data.json'''			,      help='''Input data JSON file.'''	)
      parser.add_argument('''pred_file'''			,      metavar='''pred.json'''			,      help='''Model predictions.'''	)
      parser.add_argument(
          '''--out-file'''			,      '''-o'''			,      metavar='''eval.json'''			,      help='''Write accuracy metrics to file (default is stdout).'''	)
      parser.add_argument(
          '''--na-prob-file'''			,      '''-n'''			,      metavar='''na_prob.json'''			,      help='''Model estimates of probability of no answer.'''	)
      parser.add_argument(
          '''--na-prob-thresh'''			,      '''-t'''			,      type=lowerCAmelCase_			,      default=1.0			,      help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''			,      )
      parser.add_argument(
          '''--out-image-dir'''			,      '''-p'''			,      metavar='''out_images'''			,      default=lowerCAmelCase_			,      help='''Save precision-recall curves to directory.'''	)
      parser.add_argument('''--verbose'''			,      '''-v'''			,      action='''store_true'''	)
      if len(sys.argv	) == 1:
            parser.print_help()
            sys.exit(1	)
      return parser.parse_args()




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	List[str]					     =     {}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        _snake_case		:	Union[str, Any]					     =     bool(qa['''answers''']['''text''']	)
      return qid_to_has_ans




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      def remove_articles(lowerCAmelCase_	):
            return ARTICLES_REGEX.sub(''' '''			,      lowerCAmelCase_	)

      def white_space_fix(lowerCAmelCase_	):
            return " ".join(text.split()	)

      def remove_punc(lowerCAmelCase_	):
            _snake_case		:	List[str]					     =     set(string.punctuation	)
            return "".join(ch for ch in text if ch not in exclude	)

      def lower(lowerCAmelCase_	):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase_	)	)	)	)




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      if not s:
            return []
      return normalize_answer(lowerCAmelCase_	).split()




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      return int(normalize_answer(lowerCAmelCase_	) == normalize_answer(lowerCAmelCase_	)	)




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Tuple					     =     get_tokens(lowerCAmelCase_	)
      _snake_case		:	int					     =     get_tokens(lowerCAmelCase_	)
      _snake_case		:	str					     =     collections.Counter(lowerCAmelCase_	) & collections.Counter(lowerCAmelCase_	)
      _snake_case		:	List[Any]					     =     sum(common.values()	)
      if len(lowerCAmelCase_	) == 0 or len(lowerCAmelCase_	) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks	)
      if num_same == 0:
            return 0
      _snake_case		:	Union[str, Any]					     =     1.0 * num_same / len(lowerCAmelCase_	)
      _snake_case		:	Any					     =     1.0 * num_same / len(lowerCAmelCase_	)
      _snake_case		:	List[Any]					     =     (2 * precision * recall) / (precision + recall)
      return fa




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	int					     =     {}
      _snake_case		:	Optional[int]					     =     {}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        _snake_case		:	str					     =     qa['''id''']
                        _snake_case		:	List[Any]					     =     [t for t in qa['''answers''']['''text'''] if normalize_answer(lowerCAmelCase_	)]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              _snake_case		:	Any					     =     ['''''']
                        if qid not in preds:
                              print(f'''Missing prediction for {qid}'''	)
                              continue
                        _snake_case		:	Any					     =     preds[qid]
                        # Take max over all gold answers
                        _snake_case		:	Union[str, Any]					     =     max(compute_exact(lowerCAmelCase_			,      lowerCAmelCase_	) for a in gold_answers	)
                        _snake_case		:	Dict					     =     max(compute_fa(lowerCAmelCase_			,      lowerCAmelCase_	) for a in gold_answers	)
      return exact_scores, fa_scores




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	int					     =     {}
      for qid, s in scores.items():
            _snake_case		:	Any					     =     na_probs[qid] > na_prob_thresh
            if pred_na:
                  _snake_case		:	Optional[Any]					     =     float(not qid_to_has_ans[qid]	)
            else:
                  _snake_case		:	int					     =     s
      return new_scores




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_=None	):





      """simple docstring"""



      if not qid_list:
            _snake_case		:	List[Any]					     =     len(lowerCAmelCase_	)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores.values()	) / total),
                    ('''f1''', 100.0 * sum(fa_scores.values()	) / total),
                    ('''total''', total),
                ]	)
      else:
            _snake_case		:	Tuple					     =     len(lowerCAmelCase_	)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list	) / total),
                    ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list	) / total),
                    ('''total''', total),
                ]	)




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      for k in new_eval:
            _snake_case		:	Dict					     =     new_eval[k]




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      plt.step(lowerCAmelCase_			,      lowerCAmelCase_			,      color='''b'''			,      alpha=0.2			,      where='''post'''	)
      plt.fill_between(lowerCAmelCase_			,      lowerCAmelCase_			,      step='''post'''			,      alpha=0.2			,      color='''b'''	)
      plt.xlabel('''Recall'''	)
      plt.ylabel('''Precision'''	)
      plt.xlim([0.0, 1.05]	)
      plt.ylim([0.0, 1.05]	)
      plt.title(lowerCAmelCase_	)
      plt.savefig(lowerCAmelCase_	)
      plt.clf()




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_=None			,      lowerCAmelCase_=None	):





      """simple docstring"""



      _snake_case		:	Union[str, Any]					     =     sorted(lowerCAmelCase_			,      key=lambda lowerCAmelCase_	: na_probs[k]	)
      _snake_case		:	Any					     =     0.0
      _snake_case		:	Optional[Any]					     =     1.0
      _snake_case		:	str					     =     0.0
      _snake_case		:	List[str]					     =     [1.0]
      _snake_case		:	str					     =     [0.0]
      _snake_case		:	Optional[Any]					     =     0.0
      for i, qid in enumerate(lowerCAmelCase_	):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            _snake_case		:	str					     =     true_pos / float(i + 1	)
            _snake_case		:	List[str]					     =     true_pos / float(lowerCAmelCase_	)
            if i == len(lowerCAmelCase_	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(lowerCAmelCase_	)
                  recalls.append(lowerCAmelCase_	)
      if out_image:
            plot_pr_curve(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	)
      return {"ap": 100.0 * avg_prec}




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      if out_image_dir and not os.path.exists(lowerCAmelCase_	):
            os.makedirs(lowerCAmelCase_	)
      _snake_case		:	str					     =     sum(1 for v in qid_to_has_ans.values() if v	)
      if num_true_pos == 0:
            return
      _snake_case		:	List[str]					     =     make_precision_recall_eval(
          lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      out_image=os.path.join(lowerCAmelCase_			,      '''pr_exact.png'''	)			,      title='''Precision-Recall curve for Exact Match score'''			,      )
      _snake_case		:	Any					     =     make_precision_recall_eval(
          lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      out_image=os.path.join(lowerCAmelCase_			,      '''pr_f1.png'''	)			,      title='''Precision-Recall curve for F1 score'''			,      )
      _snake_case		:	Any					     =     {k: float(lowerCAmelCase_	) for k, v in qid_to_has_ans.items()}
      _snake_case		:	Optional[int]					     =     make_precision_recall_eval(
          lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      out_image=os.path.join(lowerCAmelCase_			,      '''pr_oracle.png'''	)			,      title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''			,      )
      merge_eval(lowerCAmelCase_			,      lowerCAmelCase_			,      '''pr_exact'''	)
      merge_eval(lowerCAmelCase_			,      lowerCAmelCase_			,      '''pr_f1'''	)
      merge_eval(lowerCAmelCase_			,      lowerCAmelCase_			,      '''pr_oracle'''	)




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      if not qid_list:
            return
      _snake_case		:	int					     =     [na_probs[k] for k in qid_list]
      _snake_case		:	List[Any]					     =     np.ones_like(lowerCAmelCase_	) / float(len(lowerCAmelCase_	)	)
      plt.hist(lowerCAmelCase_			,      weights=lowerCAmelCase_			,      bins=20			,      range=(0.0, 1.0)	)
      plt.xlabel('''Model probability of no-answer'''	)
      plt.ylabel('''Proportion of dataset'''	)
      plt.title(f'''Histogram of no-answer probability: {name}'''	)
      plt.savefig(os.path.join(lowerCAmelCase_			,      f'''na_prob_hist_{name}.png'''	)	)
      plt.clf()




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Tuple					     =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
      _snake_case		:	str					     =     num_no_ans
      _snake_case		:	int					     =     cur_score
      _snake_case		:	Union[str, Any]					     =     0.0
      _snake_case		:	List[str]					     =     sorted(lowerCAmelCase_			,      key=lambda lowerCAmelCase_	: na_probs[k]	)
      for i, qid in enumerate(lowerCAmelCase_	):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  _snake_case		:	Optional[Any]					     =     scores[qid]
            else:
                  if preds[qid]:
                        _snake_case		:	Optional[Any]					     =     -1
                  else:
                        _snake_case		:	str					     =     0
            cur_score += diff
            if cur_score > best_score:
                  _snake_case		:	int					     =     cur_score
                  _snake_case		:	int					     =     na_probs[qid]
      return 100.0 * best_score / len(lowerCAmelCase_	), best_thresh




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Dict					     =     find_best_thresh(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	)
      _snake_case		:	str					     =     find_best_thresh(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	)
      _snake_case		:	str					     =     best_exact
      _snake_case		:	Any					     =     exact_thresh
      _snake_case		:	Union[str, Any]					     =     best_fa
      _snake_case		:	List[str]					     =     fa_thresh




def    _a    (							):





      """simple docstring"""



      with open(OPTS.data_file	) as f:
            _snake_case		:	str					     =     json.load(lowerCAmelCase_	)
            _snake_case		:	List[str]					     =     dataset_json['''data''']
      with open(OPTS.pred_file	) as f:
            _snake_case		:	List[Any]					     =     json.load(lowerCAmelCase_	)
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file	) as f:
                  _snake_case		:	List[Any]					     =     json.load(lowerCAmelCase_	)
      else:
            _snake_case		:	List[Any]					     =     {k: 0.0 for k in preds}
      _snake_case		:	int					     =     make_qid_to_has_ans(lowerCAmelCase_	)  # maps qid to True/False
      _snake_case		:	Optional[int]					     =     [k for k, v in qid_to_has_ans.items() if v]
      _snake_case		:	Any					     =     [k for k, v in qid_to_has_ans.items() if not v]
      _snake_case		:	Union[str, Any]					     =     get_raw_scores(lowerCAmelCase_			,      lowerCAmelCase_	)
      _snake_case		:	Union[str, Any]					     =     apply_no_ans_threshold(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      OPTS.na_prob_thresh	)
      _snake_case		:	Dict					     =     apply_no_ans_threshold(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      OPTS.na_prob_thresh	)
      _snake_case		:	Any					     =     make_eval_dict(lowerCAmelCase_			,      lowerCAmelCase_	)
      if has_ans_qids:
            _snake_case		:	Optional[int]					     =     make_eval_dict(lowerCAmelCase_			,      lowerCAmelCase_			,      qid_list=lowerCAmelCase_	)
            merge_eval(lowerCAmelCase_			,      lowerCAmelCase_			,      '''HasAns'''	)
      if no_ans_qids:
            _snake_case		:	int					     =     make_eval_dict(lowerCAmelCase_			,      lowerCAmelCase_			,      qid_list=lowerCAmelCase_	)
            merge_eval(lowerCAmelCase_			,      lowerCAmelCase_			,      '''NoAns'''	)
      if OPTS.na_prob_file:
            find_all_best_thresh(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	)
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      OPTS.out_image_dir	)
            histogram_na_prob(lowerCAmelCase_			,      lowerCAmelCase_			,      OPTS.out_image_dir			,      '''hasAns'''	)
            histogram_na_prob(lowerCAmelCase_			,      lowerCAmelCase_			,      OPTS.out_image_dir			,      '''noAns'''	)
      if OPTS.out_file:
            with open(OPTS.out_file			,      '''w'''	) as f:
                  json.dump(lowerCAmelCase_			,      lowerCAmelCase_	)
      else:
            print(json.dumps(lowerCAmelCase_			,      indent=2	)	)


if __name__ == "__main__":
   UpperCAmelCase					:		Any		=							parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use('Agg')
      import matplotlib.pyplot as plt
   main()









'''simple docstring'''

from random import randint
from tempfile import TemporaryFile

import numpy as np




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	List[Any]					     =     0
      if start < end:
            _snake_case		:	List[Any]					     =     randint(lowerCAmelCase_			,      lowerCAmelCase_	)
            _snake_case		:	Any					     =     a[end]
            _snake_case		:	List[str]					     =     a[pivot]
            _snake_case		:	Optional[int]					     =     temp

            _snake_case     ,							_snake_case		:	List[Any]					     =     _in_place_partition(lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	)
            count += _in_place_quick_sort(lowerCAmelCase_			,      lowerCAmelCase_			,      p - 1	)
            count += _in_place_quick_sort(lowerCAmelCase_			,      p + 1			,      lowerCAmelCase_	)
      return count




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Optional[Any]					     =     0
      _snake_case		:	Optional[int]					     =     randint(lowerCAmelCase_			,      lowerCAmelCase_	)
      _snake_case		:	Tuple					     =     a[end]
      _snake_case		:	Optional[Any]					     =     a[pivot]
      _snake_case		:	Union[str, Any]					     =     temp
      _snake_case		:	Union[str, Any]					     =     start - 1
      for index in range(lowerCAmelCase_			,      lowerCAmelCase_	):
            count += 1
            if a[index] < a[end]:  # check if current val is less than pivot value
                  _snake_case		:	Optional[int]					     =     new_pivot_index + 1
                  _snake_case		:	Optional[Any]					     =     a[new_pivot_index]
                  _snake_case		:	Tuple					     =     a[index]
                  _snake_case		:	str					     =     temp

      _snake_case		:	Any					     =     a[new_pivot_index + 1]
      _snake_case		:	str					     =     a[end]
      _snake_case		:	Optional[int]					     =     temp
      return new_pivot_index + 1, count


UpperCAmelCase					:		Dict		=							TemporaryFile()
UpperCAmelCase					:		Dict		=							1_0_0  # 1000 elements are to be sorted


UpperCAmelCase,       UpperCAmelCase					:		str		=							0, 1  # mean and standard deviation
UpperCAmelCase					:		Optional[Any]		=							np.random.normal(mu, sigma, p)
np.save(outfile, X)
print('The array is')
print(X)


outfile.seek(0)  # using the same array
UpperCAmelCase					:		int		=							np.load(outfile)
UpperCAmelCase					:		Optional[int]		=							len(M) - 1
UpperCAmelCase					:		str		=							_in_place_quick_sort(M, 0, r)

print(
    'No of Comparisons for 100 elements selected from a standard normal distribution'
    'is :'
)
print(z)


