


import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCamelCase        =     logging.get_logger(__name__)



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     List[Any]										=    os.path.abspath(__UpperCamelCase )
     logger.info(F'''Converting TensorFlow checkpoint from {tf_path}''' )
     # Load weights from TF model
     a_    :     Any										=    tf.train.list_variables(__UpperCamelCase )
     a_    :     Any										=    []
     a_    :     Dict										=    []
     a_    :     List[Any]										=    []
     for full_name, shape in init_vars:
          # logger.info(f"Loading TF weight {name} with shape {shape}")
          a_    :     str										=    full_name.split("""/""" )
          if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
               logger.info(F'''Skipping non-model layer {full_name}''' )
               continue
          if "optimizer" in full_name:
               logger.info(F'''Skipping optimization layer {full_name}''' )
               continue
          if name[0] == "model":
               # ignore initial 'model'
               a_    :     Any										=    name[1:]
          # figure out how many levels deep the name is
          a_    :     str										=    0
          for _name in name:
               if _name.startswith("""layer_with_weights""" ):
                    depth += 1
               else:
                    break
          layer_depth.append(__UpperCamelCase )
          # read data
          a_    :     Tuple										=    tf.train.load_variable(__UpperCamelCase  ,  __UpperCamelCase )
          names.append("""/""".join(__UpperCamelCase ) )
          arrays.append(__UpperCamelCase )
     logger.info(F'''Read a total of {len(__UpperCamelCase ):,} layers''' )

     # Sanity check
     if len(set(__UpperCamelCase ) ) != 1:
          raise ValueError(F'''Found layer names with different depths (layer depth {list(set(__UpperCamelCase ) )})''' )
     a_    :     Optional[Any]										=    list(set(__UpperCamelCase ) )[0]
     if layer_depth != 1:
          raise ValueError(
              """The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP"""
              """ heads.""" )

     # convert layers
     logger.info("""Converting weights...""" )
     for full_name, array in zip(__UpperCamelCase  ,  __UpperCamelCase ):
          a_    :     str										=    full_name.split("""/""" )
          a_    :     Dict										=    model
          a_    :     List[Any]										=    []
          for i, m_name in enumerate(__UpperCamelCase ):
               if m_name == ".ATTRIBUTES":
                    # variable names end with .ATTRIBUTES/VARIABLE_VALUE
                    break
               if m_name.startswith("""layer_with_weights""" ):
                    a_    :     List[Any]										=    int(m_name.split("""-""" )[-1] )
                    if layer_num <= 2:
                         # embedding layers
                         # layer_num 0: word_embeddings
                         # layer_num 1: position_embeddings
                         # layer_num 2: token_type_embeddings
                         continue
                    elif layer_num == 3:
                         # embedding LayerNorm
                         trace.extend(["""embeddings""", """LayerNorm"""] )
                         a_    :     int										=    getattr(__UpperCamelCase  ,  """embeddings""" )
                         a_    :     Tuple										=    getattr(__UpperCamelCase  ,  """LayerNorm""" )
                    elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
                         # encoder layers
                         trace.extend(["""encoder""", """layer""", str(layer_num - 4 )] )
                         a_    :     Dict										=    getattr(__UpperCamelCase  ,  """encoder""" )
                         a_    :     str										=    getattr(__UpperCamelCase  ,  """layer""" )
                         a_    :     Dict										=    pointer[layer_num - 4]
                    elif layer_num == config.num_hidden_layers + 4:
                         # pooler layer
                         trace.extend(["""pooler""", """dense"""] )
                         a_    :     Tuple										=    getattr(__UpperCamelCase  ,  """pooler""" )
                         a_    :     List[Any]										=    getattr(__UpperCamelCase  ,  """dense""" )
               elif m_name == "embeddings":
                    trace.append("""embeddings""" )
                    a_    :     Tuple										=    getattr(__UpperCamelCase  ,  """embeddings""" )
                    if layer_num == 0:
                         trace.append("""word_embeddings""" )
                         a_    :     Tuple										=    getattr(__UpperCamelCase  ,  """word_embeddings""" )
                    elif layer_num == 1:
                         trace.append("""position_embeddings""" )
                         a_    :     Optional[Any]										=    getattr(__UpperCamelCase  ,  """position_embeddings""" )
                    elif layer_num == 2:
                         trace.append("""token_type_embeddings""" )
                         a_    :     str										=    getattr(__UpperCamelCase  ,  """token_type_embeddings""" )
                    else:
                         raise ValueError(F'''Unknown embedding layer with name {full_name}''' )
                    trace.append("""weight""" )
                    a_    :     Union[str, Any]										=    getattr(__UpperCamelCase  ,  """weight""" )
               elif m_name == "_attention_layer":
                    # self-attention layer
                    trace.extend(["""attention""", """self"""] )
                    a_    :     List[Any]										=    getattr(__UpperCamelCase  ,  """attention""" )
                    a_    :     List[str]										=    getattr(__UpperCamelCase  ,  """self""" )
               elif m_name == "_attention_layer_norm":
                    # output attention norm
                    trace.extend(["""attention""", """output""", """LayerNorm"""] )
                    a_    :     Optional[int]										=    getattr(__UpperCamelCase  ,  """attention""" )
                    a_    :     Any										=    getattr(__UpperCamelCase  ,  """output""" )
                    a_    :     Optional[Any]										=    getattr(__UpperCamelCase  ,  """LayerNorm""" )
               elif m_name == "_attention_output_dense":
                    # output attention dense
                    trace.extend(["""attention""", """output""", """dense"""] )
                    a_    :     int										=    getattr(__UpperCamelCase  ,  """attention""" )
                    a_    :     str										=    getattr(__UpperCamelCase  ,  """output""" )
                    a_    :     Dict										=    getattr(__UpperCamelCase  ,  """dense""" )
               elif m_name == "_output_dense":
                    # output dense
                    trace.extend(["""output""", """dense"""] )
                    a_    :     Tuple										=    getattr(__UpperCamelCase  ,  """output""" )
                    a_    :     List[str]										=    getattr(__UpperCamelCase  ,  """dense""" )
               elif m_name == "_output_layer_norm":
                    # output dense
                    trace.extend(["""output""", """LayerNorm"""] )
                    a_    :     Optional[int]										=    getattr(__UpperCamelCase  ,  """output""" )
                    a_    :     Dict										=    getattr(__UpperCamelCase  ,  """LayerNorm""" )
               elif m_name == "_key_dense":
                    # attention key
                    trace.append("""key""" )
                    a_    :     str										=    getattr(__UpperCamelCase  ,  """key""" )
               elif m_name == "_query_dense":
                    # attention query
                    trace.append("""query""" )
                    a_    :     Optional[Any]										=    getattr(__UpperCamelCase  ,  """query""" )
               elif m_name == "_value_dense":
                    # attention value
                    trace.append("""value""" )
                    a_    :     Optional[int]										=    getattr(__UpperCamelCase  ,  """value""" )
               elif m_name == "_intermediate_dense":
                    # attention intermediate dense
                    trace.extend(["""intermediate""", """dense"""] )
                    a_    :     Union[str, Any]										=    getattr(__UpperCamelCase  ,  """intermediate""" )
                    a_    :     Any										=    getattr(__UpperCamelCase  ,  """dense""" )
               elif m_name == "_output_layer_norm":
                    # output layer norm
                    trace.append("""output""" )
                    a_    :     Dict										=    getattr(__UpperCamelCase  ,  """output""" )
               # weights & biases
               elif m_name in ["bias", "beta"]:
                    trace.append("""bias""" )
                    a_    :     Tuple										=    getattr(__UpperCamelCase  ,  """bias""" )
               elif m_name in ["kernel", "gamma"]:
                    trace.append("""weight""" )
                    a_    :     str										=    getattr(__UpperCamelCase  ,  """weight""" )
               else:
                    logger.warning(F'''Ignored {m_name}''' )
        # for certain layers reshape is necessary
          a_    :     str										=    """.""".join(__UpperCamelCase )
          if re.match(r"""(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)"""  ,  __UpperCamelCase ) or re.match(
              r"""(\S+)\.attention\.output\.dense\.weight"""  ,  __UpperCamelCase ):
               a_    :     Any										=    array.reshape(pointer.data.shape )
          if "kernel" in full_name:
               a_    :     Optional[int]										=    array.transpose()
          if pointer.shape == array.shape:
               a_    :     str										=    torch.from_numpy(__UpperCamelCase )
          else:
               raise ValueError(
                   F'''Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:'''
                   F''' {array.shape}''' )
          logger.info(F'''Successfully set variable {full_name} to PyTorch layer {trace}''' )
     return model



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     # Instantiate model
     logger.info(F'''Loading model based on config from {config_path}...''' )
     a_    :     Optional[Any]										=    BertConfig.from_json_file(__UpperCamelCase )
     a_    :     List[str]										=    BertModel(__UpperCamelCase )

     # Load weights from checkpoint
     logger.info(F'''Loading weights from checkpoint {tf_checkpoint_path}...''' )
     load_tfa_weights_in_bert(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )

     # Save pytorch-model
     logger.info(F'''Saving PyTorch model to {pytorch_dump_path}...''' )
     torch.save(model.state_dict()  ,  __UpperCamelCase )


if __name__ == "__main__":
       __lowerCamelCase        =     argparse.ArgumentParser()
       parser.add_argument(
           '''--tf_checkpoint_path''', type=str, required=True, help='''Path to the TensorFlow 2.x checkpoint path.'''
       )
       parser.add_argument(
           '''--bert_config_file''',
           type=str,
           required=True,
           help='''The config json file corresponding to the BERT model. This specifies the model architecture.''',
       )
       parser.add_argument(
           '''--pytorch_dump_path''',
           type=str,
           required=True,
           help='''Path to the output PyTorch model (must include filename).''',
       )
       __lowerCamelCase        =     parser.parse_args()
       convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)








import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase        =     re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__lowerCamelCase        =     None



def  _a			(						):
     a_    :     Tuple										=    argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
     parser.add_argument("""data_file"""  ,  metavar="""data.json"""  ,  help="""Input data JSON file.""" )
     parser.add_argument("""pred_file"""  ,  metavar="""pred.json"""  ,  help="""Model predictions.""" )
     parser.add_argument(
         """--out-file"""  ,  """-o"""  ,  metavar="""eval.json"""  ,  help="""Write accuracy metrics to file (default is stdout).""" )
     parser.add_argument(
         """--na-prob-file"""  ,  """-n"""  ,  metavar="""na_prob.json"""  ,  help="""Model estimates of probability of no answer.""" )
     parser.add_argument(
         """--na-prob-thresh"""  ,  """-t"""  ,  type=__UpperCamelCase  ,  default=1.0  ,  help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""  ,  )
     parser.add_argument(
         """--out-image-dir"""  ,  """-p"""  ,  metavar="""out_images"""  ,  default=__UpperCamelCase  ,  help="""Save precision-recall curves to directory.""" )
     parser.add_argument("""--verbose"""  ,  """-v"""  ,  action="""store_true""" )
     if len(sys.argv ) == 1:
          parser.print_help()
          sys.exit(1 )
     return parser.parse_args()



def  _a			(						__UpperCamelCase ):
     a_    :     Union[str, Any]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     Union[str, Any]										=    bool(qa["""answers"""]["""text"""] )
     return qid_to_has_ans



def  _a			(						__UpperCamelCase ):

     def remove_articles(__UpperCamelCase ):
          return ARTICLES_REGEX.sub(""" """  ,  __UpperCamelCase )

     def white_space_fix(__UpperCamelCase ):
          return " ".join(text.split() )

     def remove_punc(__UpperCamelCase ):
          a_    :     List[str]										=    set(string.punctuation )
          return "".join(ch for ch in text if ch not in exclude )

     def lower(__UpperCamelCase ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase ) ) ) )



def  _a			(						__UpperCamelCase ):
     if not s:
          return []
     return normalize_answer(__UpperCamelCase ).split()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     return int(normalize_answer(__UpperCamelCase ) == normalize_answer(__UpperCamelCase ) )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     List[Any]										=    get_tokens(__UpperCamelCase )
     a_    :     str										=    get_tokens(__UpperCamelCase )
     a_    :     Tuple										=    collections.Counter(__UpperCamelCase ) & collections.Counter(__UpperCamelCase )
     a_    :     str										=    sum(common.values() )
     if len(__UpperCamelCase ) == 0 or len(__UpperCamelCase ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks )
     if num_same == 0:
          return 0
     a_    :     Any										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     Optional[int]										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     List[Any]										=    (2 * precision * recall) / (precision + recall)
     return fa



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Tuple										=    {}
     a_    :     Optional[int]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     str										=    qa["""id"""]
                    a_    :     Dict										=    [t for t in qa["""answers"""]["""text"""] if normalize_answer(__UpperCamelCase )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         a_    :     Union[str, Any]										=    [""""""]
                    if qid not in preds:
                         print(F'''Missing prediction for {qid}''' )
                         continue
                    a_    :     int										=    preds[qid]
                    # Take max over all gold answers
                    a_    :     Union[str, Any]										=    max(compute_exact(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
                    a_    :     int										=    max(compute_fa(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
     return exact_scores, fa_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[Any]										=    {}
     for qid, s in scores.items():
          a_    :     Optional[int]										=    na_probs[qid] > na_prob_thresh
          if pred_na:
               a_    :     List[Any]										=    float(not qid_to_has_ans[qid] )
          else:
               a_    :     List[Any]										=    s
     return new_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None ):
     if not qid_list:
          a_    :     List[Any]										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores.values() ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores.values() ) / total),
                  ("""total""", total),
              ] )
     else:
          a_    :     Tuple										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores[k] for k in qid_list ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores[k] for k in qid_list ) / total),
                  ("""total""", total),
              ] )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     for k in new_eval:
          a_    :     Any										=    new_eval[k]



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     plt.step(__UpperCamelCase  ,  __UpperCamelCase  ,  color="""b"""  ,  alpha=0.2  ,  where="""post""" )
     plt.fill_between(__UpperCamelCase  ,  __UpperCamelCase  ,  step="""post"""  ,  alpha=0.2  ,  color="""b""" )
     plt.xlabel("""Recall""" )
     plt.ylabel("""Precision""" )
     plt.xlim([0.0, 1.05] )
     plt.ylim([0.0, 1.05] )
     plt.title(__UpperCamelCase )
     plt.savefig(__UpperCamelCase )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None  ,  __UpperCamelCase=None ):
     a_    :     List[Any]										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     a_    :     int										=    0.0
     a_    :     Tuple										=    1.0
     a_    :     Union[str, Any]										=    0.0
     a_    :     List[str]										=    [1.0]
     a_    :     int										=    [0.0]
     a_    :     Optional[int]										=    0.0
     for i, qid in enumerate(__UpperCamelCase ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          a_    :     Union[str, Any]										=    true_pos / float(i + 1 )
          a_    :     Tuple										=    true_pos / float(__UpperCamelCase )
          if i == len(__UpperCamelCase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(__UpperCamelCase )
               recalls.append(__UpperCamelCase )
     if out_image:
          plot_pr_curve(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     return {"ap": 1_00.0 * avg_prec}



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if out_image_dir and not os.path.exists(__UpperCamelCase ):
          os.makedirs(__UpperCamelCase )
     a_    :     Any										=    sum(1 for v in qid_to_has_ans.values() if v )
     if num_true_pos == 0:
          return
     a_    :     Union[str, Any]										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_exact.png""" )  ,  title="""Precision-Recall curve for Exact Match score"""  ,  )
     a_    :     Any										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_f1.png""" )  ,  title="""Precision-Recall curve for F1 score"""  ,  )
     a_    :     int										=    {k: float(__UpperCamelCase ) for k, v in qid_to_has_ans.items()}
     a_    :     Dict										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_oracle.png""" )  ,  title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""  ,  )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_exact""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_f1""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_oracle""" )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if not qid_list:
          return
     a_    :     List[Any]										=    [na_probs[k] for k in qid_list]
     a_    :     List[Any]										=    np.ones_like(__UpperCamelCase ) / float(len(__UpperCamelCase ) )
     plt.hist(__UpperCamelCase  ,  weights=__UpperCamelCase  ,  bins=2_0  ,  range=(0.0, 1.0) )
     plt.xlabel("""Model probability of no-answer""" )
     plt.ylabel("""Proportion of dataset""" )
     plt.title(F'''Histogram of no-answer probability: {name}''' )
     plt.savefig(os.path.join(__UpperCamelCase  ,  F'''na_prob_hist_{name}.png''' ) )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[int]										=    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
     a_    :     Tuple										=    num_no_ans
     a_    :     List[Any]										=    cur_score
     a_    :     List[Any]										=    0.0
     a_    :     int										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     for i, qid in enumerate(__UpperCamelCase ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               a_    :     Optional[Any]										=    scores[qid]
          else:
               if preds[qid]:
                    a_    :     Optional[int]										=    -1
               else:
                    a_    :     Optional[int]										=    0
          cur_score += diff
          if cur_score > best_score:
               a_    :     str										=    cur_score
               a_    :     int										=    na_probs[qid]
     return 1_00.0 * best_score / len(__UpperCamelCase ), best_thresh



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_			,			a_    :     Optional[int]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_			,			a_    :     Optional[Any]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_    :     Dict										=    best_exact
     a_    :     Optional[Any]										=    exact_thresh
     a_    :     Optional[Any]										=    best_fa
     a_    :     Union[str, Any]										=    fa_thresh



def  _a			(						):
     with open(OPTS.data_file ) as f:
          a_    :     str										=    json.load(__UpperCamelCase )
          a_    :     Optional[int]										=    dataset_json["""data"""]
     with open(OPTS.pred_file ) as f:
          a_    :     Dict										=    json.load(__UpperCamelCase )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file ) as f:
               a_    :     List[str]										=    json.load(__UpperCamelCase )
     else:
          a_    :     List[str]										=    {k: 0.0 for k in preds}
     a_    :     Any										=    make_qid_to_has_ans(__UpperCamelCase )  # maps qid to True/False
     a_    :     Any										=    [k for k, v in qid_to_has_ans.items() if v]
     a_    :     Union[str, Any]										=    [k for k, v in qid_to_has_ans.items() if not v]
     a_			,			a_    :     List[str]										=    get_raw_scores(__UpperCamelCase  ,  __UpperCamelCase )
     a_    :     int										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     Optional[int]										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase )
     if has_ans_qids:
          a_    :     Dict										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """HasAns""" )
     if no_ans_qids:
          a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """NoAns""" )
     if OPTS.na_prob_file:
          find_all_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """hasAns""" )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """noAns""" )
     if OPTS.out_file:
          with open(OPTS.out_file  ,  """w""" ) as f:
               json.dump(__UpperCamelCase  ,  __UpperCamelCase )
     else:
          print(json.dumps(__UpperCamelCase  ,  indent=2 ) )


if __name__ == "__main__":
       __lowerCamelCase        =     parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('''Agg''')
              import matplotlib.pyplot as plt
       main()



