




'''simple docstring'''


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase_ 		=      re.compile(r"\b(a|an|the)\b", re.UNICODE)

UpperCAmelCase_ 		=      None


def 						SCREAMING_SNAKE_CASE       (      ):
  __a			=					argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'			)
  parser.add_argument('data_file'				,					metavar='data.json'				,					help='Input data JSON file.'			)
  parser.add_argument('pred_file'				,					metavar='pred.json'				,					help='Model predictions.'			)
  parser.add_argument(
      '--out-file'				,					'-o'				,					metavar='eval.json'				,					help='Write accuracy metrics to file (default is stdout).'			)
  parser.add_argument(
      '--na-prob-file'				,					'-n'				,					metavar='na_prob.json'				,					help='Model estimates of probability of no answer.'			)
  parser.add_argument(
      '--na-prob-thresh'				,					'-t'				,					type=a_				,					default=1.0				,					help='Predict "" if no-answer probability exceeds this (default = 1.0).'				,					)
  parser.add_argument(
      '--out-image-dir'				,					'-p'				,					metavar='out_images'				,					default=a_				,					help='Save precision-recall curves to directory.'			)
  parser.add_argument('--verbose'				,					'-v'				,					action='store_true'			)
  if len(sys.argv			) == 1:
    parser.print_help()
    sys.exit(1			)
  return parser.parse_args()


def 						SCREAMING_SNAKE_CASE       (      a_	:							str			):
  __a			=					{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        __a			=					bool(qa['answers']['text']			)
  return qid_to_has_ans


def 						SCREAMING_SNAKE_CASE       (      a_	:							List[Any]			):

  def remove_articles(a_	:							Union[str, Any]			):
    return ARTICLES_REGEX.sub(' '				,					a_			)

  def white_space_fix(a_	:							Optional[Any]			):
    return " ".join(text.split()			)

  def remove_punc(a_	:							Optional[Any]			):
    __a			=					set(string.punctuation			)
    return "".join(ch for ch in text if ch not in exclude			)

  def lower(a_	:							List[Any]			):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(a_			)			)			)			)


def 						SCREAMING_SNAKE_CASE       (      a_	:							Any			):
  if not s:
    return []
  return normalize_answer(a_			).split()


def 						SCREAMING_SNAKE_CASE       (      a_	:							Dict				,					a_	:							str			):
  return int(normalize_answer(a_			) == normalize_answer(a_			)			)


def 						SCREAMING_SNAKE_CASE       (      a_	:							List[Any]				,					a_	:							List[Any]			):
  __a			=					get_tokens(a_			)
  __a			=					get_tokens(a_			)
  __a			=					collections.Counter(a_			) & collections.Counter(a_			)
  __a			=					sum(common.values()			)
  if len(a_			) == 0 or len(a_			) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks			)
  if num_same == 0:
    return 0
  __a			=					1.0 * num_same / len(a_			)
  __a			=					1.0 * num_same / len(a_			)
  __a			=					(2 * precision * recall) / (precision + recall)
  return fa


def 						SCREAMING_SNAKE_CASE       (      a_	:							Union[str, Any]				,					a_	:							List[Any]			):
  __a			=					{}
  __a			=					{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        __a			=					qa['id']
        __a			=					[t for t in qa['answers']['text'] if normalize_answer(a_			)]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          __a			=					['']
        if qid not in preds:
          print(f"Missing prediction for {qid}"			)
          continue
        __a			=					preds[qid]
        # Take max over all gold answers
        __a			=					max(compute_exact(a_				,					a_			) for a in gold_answers			)
        __a			=					max(compute_fa(a_				,					a_			) for a in gold_answers			)
  return exact_scores, fa_scores


def 						SCREAMING_SNAKE_CASE       (      a_	:							Union[str, Any]				,					a_	:							List[Any]				,					a_	:							List[str]				,					a_	:							Optional[Any]			):
  __a			=					{}
  for qid, s in scores.items():
    __a			=					na_probs[qid] > na_prob_thresh
    if pred_na:
      __a			=					float(not qid_to_has_ans[qid]			)
    else:
      __a			=					s
  return new_scores


def 						SCREAMING_SNAKE_CASE       (      a_	:							Optional[int]				,					a_	:							Optional[int]				,					a_	:							Optional[int]=None			):
  if not qid_list:
    __a			=					len(a_			)
    return collections.OrderedDict(
        [
            ('exact', 1_00.0 * sum(exact_scores.values()			) / total),
            ('f1', 1_00.0 * sum(fa_scores.values()			) / total),
            ('total', total),
        ]			)
  else:
    __a			=					len(a_			)
    return collections.OrderedDict(
        [
            ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list			) / total),
            ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list			) / total),
            ('total', total),
        ]			)


def 						SCREAMING_SNAKE_CASE       (      a_	:							Union[str, Any]				,					a_	:							Any				,					a_	:							Dict			):
  for k in new_eval:
    __a			=					new_eval[k]


def 						SCREAMING_SNAKE_CASE       (      a_	:							str				,					a_	:							Any				,					a_	:							Dict				,					a_	:							str			):
  plt.step(a_				,					a_				,					color='b'				,					alpha=0.2				,					where='post'			)
  plt.fill_between(a_				,					a_				,					step='post'				,					alpha=0.2				,					color='b'			)
  plt.xlabel('Recall'			)
  plt.ylabel('Precision'			)
  plt.xlim([0.0, 1.05]			)
  plt.ylim([0.0, 1.05]			)
  plt.title(a_			)
  plt.savefig(a_			)
  plt.clf()


def 						SCREAMING_SNAKE_CASE       (      a_	:							Dict				,					a_	:							int				,					a_	:							str				,					a_	:							Any				,					a_	:							Tuple=None				,					a_	:							Any=None			):
  __a			=					sorted(a_				,					key=lambda a_			: na_probs[k]			)
  __a			=					0.0
  __a			=					1.0
  __a			=					0.0
  __a			=					[1.0]
  __a			=					[0.0]
  __a			=					0.0
  for i, qid in enumerate(a_			):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    __a			=					true_pos / float(i + 1			)
    __a			=					true_pos / float(a_			)
    if i == len(a_			) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(a_			)
      recalls.append(a_			)
  if out_image:
    plot_pr_curve(a_				,					a_				,					a_				,					a_			)
  return {"ap": 1_00.0 * avg_prec}


def 						SCREAMING_SNAKE_CASE       (      a_	:							str				,					a_	:							Tuple				,					a_	:							Union[str, Any]				,					a_	:							Any				,					a_	:							Optional[Any]				,					a_	:							List[Any]			):
  if out_image_dir and not os.path.exists(a_			):
    os.makedirs(a_			)
  __a			=					sum(1 for v in qid_to_has_ans.values() if v			)
  if num_true_pos == 0:
    return
  __a			=					make_precision_recall_eval(
      a_				,					a_				,					a_				,					a_				,					out_image=os.path.join(a_				,					'pr_exact.png'			)				,					title='Precision-Recall curve for Exact Match score'				,					)
  __a			=					make_precision_recall_eval(
      a_				,					a_				,					a_				,					a_				,					out_image=os.path.join(a_				,					'pr_f1.png'			)				,					title='Precision-Recall curve for F1 score'				,					)
  __a			=					{k: float(a_			) for k, v in qid_to_has_ans.items()}
  __a			=					make_precision_recall_eval(
      a_				,					a_				,					a_				,					a_				,					out_image=os.path.join(a_				,					'pr_oracle.png'			)				,					title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'				,					)
  merge_eval(a_				,					a_				,					'pr_exact'			)
  merge_eval(a_				,					a_				,					'pr_f1'			)
  merge_eval(a_				,					a_				,					'pr_oracle'			)


def 						SCREAMING_SNAKE_CASE       (      a_	:							List[str]				,					a_	:							List[Any]				,					a_	:							Union[str, Any]				,					a_	:							Dict			):
  if not qid_list:
    return
  __a			=					[na_probs[k] for k in qid_list]
  __a			=					np.ones_like(a_			) / float(len(a_			)			)
  plt.hist(a_				,					weights=a_				,					bins=20				,					range=(0.0, 1.0)			)
  plt.xlabel('Model probability of no-answer'			)
  plt.ylabel('Proportion of dataset'			)
  plt.title(f"Histogram of no-answer probability: {name}"			)
  plt.savefig(os.path.join(a_				,					f"na_prob_hist_{name}.png"			)			)
  plt.clf()


def 						SCREAMING_SNAKE_CASE       (      a_	:							int				,					a_	:							str				,					a_	:							List[str]				,					a_	:							Dict			):
  __a			=					sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]			)
  __a			=					num_no_ans
  __a			=					cur_score
  __a			=					0.0
  __a			=					sorted(a_				,					key=lambda a_			: na_probs[k]			)
  for i, qid in enumerate(a_			):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      __a			=					scores[qid]
    else:
      if preds[qid]:
        __a			=					-1
      else:
        __a			=					0
    cur_score += diff
    if cur_score > best_score:
      __a			=					cur_score
      __a			=					na_probs[qid]
  return 1_00.0 * best_score / len(a_			), best_thresh


def 						SCREAMING_SNAKE_CASE       (      a_	:							Optional[int]				,					a_	:							Any				,					a_	:							int				,					a_	:							str				,					a_	:							Dict				,					a_	:							Dict			):
  __a	,       __a			=					find_best_thresh(a_				,					a_				,					a_				,					a_			)
  __a	,       __a			=					find_best_thresh(a_				,					a_				,					a_				,					a_			)
  __a			=					best_exact
  __a			=					exact_thresh
  __a			=					best_fa
  __a			=					fa_thresh


def 						SCREAMING_SNAKE_CASE       (      ):
  with open(OPTS.data_file			) as f:
    __a			=					json.load(a_			)
    __a			=					dataset_json['data']
  with open(OPTS.pred_file			) as f:
    __a			=					json.load(a_			)
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file			) as f:
      __a			=					json.load(a_			)
  else:
    __a			=					{k: 0.0 for k in preds}
  __a			=					make_qid_to_has_ans(a_			)  # maps qid to True/False
  __a			=					[k for k, v in qid_to_has_ans.items() if v]
  __a			=					[k for k, v in qid_to_has_ans.items() if not v]
  __a	,       __a			=					get_raw_scores(a_				,					a_			)
  __a			=					apply_no_ans_threshold(a_				,					a_				,					a_				,					OPTS.na_prob_thresh			)
  __a			=					apply_no_ans_threshold(a_				,					a_				,					a_				,					OPTS.na_prob_thresh			)
  __a			=					make_eval_dict(a_				,					a_			)
  if has_ans_qids:
    __a			=					make_eval_dict(a_				,					a_				,					qid_list=a_			)
    merge_eval(a_				,					a_				,					'HasAns'			)
  if no_ans_qids:
    __a			=					make_eval_dict(a_				,					a_				,					qid_list=a_			)
    merge_eval(a_				,					a_				,					'NoAns'			)
  if OPTS.na_prob_file:
    find_all_best_thresh(a_				,					a_				,					a_				,					a_				,					a_				,					a_			)
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(a_				,					a_				,					a_				,					a_				,					a_				,					OPTS.out_image_dir			)
    histogram_na_prob(a_				,					a_				,					OPTS.out_image_dir				,					'hasAns'			)
    histogram_na_prob(a_				,					a_				,					OPTS.out_image_dir				,					'noAns'			)
  if OPTS.out_file:
    with open(OPTS.out_file				,					'w'			) as f:
      json.dump(a_				,					a_			)
  else:
    print(json.dumps(a_				,					indent=2			)			)


if __name__ == "__main__":
  UpperCAmelCase_ 		=      parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
  main()









'''simple docstring'''


from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {
    "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/config.json",
    "distilbert-base-uncased-distilled-squad": (
        "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json"
    ),
    "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/config.json",
    "distilbert-base-cased-distilled-squad": (
        "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json"
    ),
    "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json",
    "distilbert-base-multilingual-cased": (
        "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json"
    ),
    "distilbert-base-uncased-finetuned-sst-2-english": (
        "https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json"
    ),
}





class    __lowercase						(  __magic_name__ ):
   _a							=      """distilbert"""
   _a							=      {
       """hidden_size""": """dim""",
       """num_attention_heads""": """n_heads""",
       """num_hidden_layers""": """n_layers""",
   }







   def __init__(	self      ,							UpperCamelCase=3_0522      ,							UpperCamelCase=512      ,							UpperCamelCase=False      ,							UpperCamelCase=6      ,							UpperCamelCase=12      ,							UpperCamelCase=768      ,							UpperCamelCase=4 * 768      ,							UpperCamelCase=0.1      ,							UpperCamelCase=0.1      ,							UpperCamelCase="gelu"      ,							UpperCamelCase=0.02      ,							UpperCamelCase=0.1      ,							UpperCamelCase=0.2      ,							UpperCamelCase=0      ,							**UpperCamelCase      ,							)				->    int:
     __a			=					vocab_size
     __a			=					max_position_embeddings
     __a			=					sinusoidal_pos_embds
     __a			=					n_layers
     __a			=					n_heads
     __a			=					dim
     __a			=					hidden_dim
     __a			=					dropout
     __a			=					attention_dropout
     __a			=					activation
     __a			=					initializer_range
     __a			=					qa_dropout
     __a			=					seq_classif_dropout
     super().__init__(**UpperCamelCase      ,							pad_token_id=UpperCamelCase      )





class    __lowercase						(  __magic_name__ ):







   @property
   def 		UpperCamelCase__			(	self      )				->    Mapping[str, Mapping[int, str]]:
     if self.task == "multiple-choice":
       __a			=					{0: 'batch', 1: 'choice', 2: 'sequence'}
     else:
       __a			=					{0: 'batch', 1: 'sequence'}
     return OrderedDict(
         [
             ('input_ids', dynamic_axis),
             ('attention_mask', dynamic_axis),
         ]      )



