


from __future__ import annotations

import unittest

from transformers import EsmConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import numpy
       import tensorflow as tf

       from transformers.models.esm.modeling_tf_esm import (
           TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
           TFEsmForMaskedLM,
           TFEsmForSequenceClassification,
           TFEsmForTokenClassification,
           TFEsmModel,
       )


class   a__	:

       def __init__(   self      :     Dict      ,							lowerCamelCase_      :     List[str]      ,							):
            a_    :     List[Any]										=    parent
            a_    :     str										=    1_3
            a_    :     Optional[int]										=    7
            a_    :     int										=    True
            a_    :     Any										=    True
            a_    :     List[Any]										=    True
            a_    :     Any										=    9_9
            a_    :     int										=    3_2
            a_    :     Tuple										=    2
            a_    :     Union[str, Any]										=    4
            a_    :     int										=    3_7
            a_    :     Optional[int]										=    "gelu"
            a_    :     Tuple										=    0.1
            a_    :     Dict										=    0.1
            a_    :     str										=    5_1_2
            a_    :     Tuple										=    1_6
            a_    :     Tuple										=    2
            a_    :     str										=    0.0_2
            a_    :     int										=    3
            a_    :     Optional[int]										=    4
            a_    :     Dict										=    None

       def 						UpperCAmelCase(   self      :     int    ):
            a_    :     Any										=    ids_tensor([self.batch_size, self.seq_length]      ,							self.vocab_size    )

            a_    :     Tuple										=    None
            if self.use_input_mask:
                 a_    :     List[Any]										=    random_attention_mask([self.batch_size, self.seq_length]    )

            a_    :     Any										=    None
            a_    :     Dict										=    None
            a_    :     Union[str, Any]										=    None
            if self.use_labels:
                 a_    :     Tuple										=    ids_tensor([self.batch_size]      ,							self.type_sequence_label_size    )
                 a_    :     List[Any]										=    ids_tensor([self.batch_size, self.seq_length]      ,							self.num_labels    )
                 a_    :     Optional[Any]										=    ids_tensor([self.batch_size]      ,							self.num_choices    )

            a_    :     Optional[Any]										=    EsmConfig(
                vocab_size=self.vocab_size      ,							hidden_size=self.hidden_size      ,							num_hidden_layers=self.num_hidden_layers      ,							pad_token_id=1      ,							num_attention_heads=self.num_attention_heads      ,							intermediate_size=self.intermediate_size      ,							hidden_act=self.hidden_act      ,							hidden_dropout_prob=self.hidden_dropout_prob      ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,							max_position_embeddings=self.max_position_embeddings      ,							type_vocab_size=self.type_vocab_size      ,							initializer_range=self.initializer_range      ,							)

            return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels

       def 						UpperCAmelCase(   self      :     int    ):
            (
                a_ 
            )    :     List[str]										=    self.prepare_config_and_inputs()

            a_    :     str										=    True
            a_    :     str										=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
            a_    :     Any										=    ids_tensor([self.batch_size, self.seq_length]      ,							vocab_size=2    )

            return (
                config,
                input_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                encoder_hidden_states,
                encoder_attention_mask,
            )

       def 						UpperCAmelCase(   self      :     List[Any]      ,							lowerCamelCase_      :     str      ,							lowerCamelCase_      :     str      ,							lowerCamelCase_      :     Tuple      ,							lowerCamelCase_      :     Tuple      ,							lowerCamelCase_      :     List[Any]      ,							lowerCamelCase_      :     Optional[int]    ):
            a_    :     Tuple										=    TFEsmModel(config=__lowerCamelCase    )
            a_    :     List[Any]										=    {"input_ids": input_ids, "attention_mask": input_mask}
            a_    :     Optional[int]										=    model(__lowerCamelCase    )

            a_    :     List[Any]										=    [input_ids, input_mask]
            a_    :     List[Any]										=    model(__lowerCamelCase    )

            a_    :     Optional[Any]										=    model(__lowerCamelCase    )

            self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, self.seq_length, self.hidden_size)    )

       def 						UpperCAmelCase(   self      :     List[str]      ,							lowerCamelCase_      :     List[str]      ,							lowerCamelCase_      :     Dict      ,							lowerCamelCase_      :     int      ,							lowerCamelCase_      :     str      ,							lowerCamelCase_      :     List[str]      ,							lowerCamelCase_      :     Optional[Any]      ,							lowerCamelCase_      :     Tuple      ,							lowerCamelCase_      :     List[str]      ,							):
            a_    :     Dict										=    True

            a_    :     Union[str, Any]										=    TFEsmModel(config=__lowerCamelCase    )
            a_    :     Optional[int]										=    {
                "input_ids": input_ids,
                "attention_mask": input_mask,
                "encoder_hidden_states": encoder_hidden_states,
                "encoder_attention_mask": encoder_attention_mask,
            }
            a_    :     str										=    model(__lowerCamelCase    )

            a_    :     Tuple										=    [input_ids, input_mask]
            a_    :     Union[str, Any]										=    model(__lowerCamelCase      ,							encoder_hidden_states=__lowerCamelCase    )

            # Also check the case where encoder outputs are not passed
            a_    :     str										=    model(__lowerCamelCase      ,							attention_mask=__lowerCamelCase    )

            self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, self.seq_length, self.hidden_size)    )

       def 						UpperCAmelCase(   self      :     str      ,							lowerCamelCase_      :     Union[str, Any]      ,							lowerCamelCase_      :     Optional[Any]      ,							lowerCamelCase_      :     Dict      ,							lowerCamelCase_      :     Union[str, Any]      ,							lowerCamelCase_      :     Union[str, Any]      ,							lowerCamelCase_      :     Union[str, Any]    ):
            a_    :     Dict										=    TFEsmForMaskedLM(config=__lowerCamelCase    )
            a_    :     Any										=    model([input_ids, input_mask]    )
            self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.seq_length, self.vocab_size)    )

       def 						UpperCAmelCase(   self      :     List[Any]      ,							lowerCamelCase_      :     Dict      ,							lowerCamelCase_      :     str      ,							lowerCamelCase_      :     Tuple      ,							lowerCamelCase_      :     Optional[int]      ,							lowerCamelCase_      :     int      ,							lowerCamelCase_      :     int    ):
            a_    :     Any										=    self.num_labels
            a_    :     Tuple										=    TFEsmForTokenClassification(config=__lowerCamelCase    )
            a_    :     Dict										=    {"input_ids": input_ids, "attention_mask": input_mask}
            a_    :     int										=    model(__lowerCamelCase    )
            self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.seq_length, self.num_labels)    )

       def 						UpperCAmelCase(   self      :     Dict    ):
            a_    :     int										=    self.prepare_config_and_inputs()
            (
                a_ 
            )    :     Tuple										=    config_and_inputs
            a_    :     int										=    {"input_ids": input_ids, "attention_mask": input_mask}
            return config, inputs_dict


@require_tf
class   a__	(						_A						,       _A						,       unittest.TestCase    ):
       lowerCamelCase__:     List[Any]							    =     (
           (
               TFEsmModel,
               TFEsmForMaskedLM,
               TFEsmForSequenceClassification,
               TFEsmForTokenClassification,
           )
           if is_tf_available()
           else ()
       )
       lowerCamelCase__:     List[str]							    =     (
           {
               """feature-extraction""": TFEsmModel,
               """fill-mask""": TFEsmForMaskedLM,
               """text-classification""": TFEsmForSequenceClassification,
               """token-classification""": TFEsmForTokenClassification,
               """zero-shot""": TFEsmForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       lowerCamelCase__:     Optional[Any]							    =     False
       lowerCamelCase__:     List[str]							    =     False

       def 						UpperCAmelCase(   self      :     int    ):
            a_    :     Any										=    TFEsmModelTester(self    )
            a_    :     Optional[int]										=    ConfigTester(self      ,							config_class=__lowerCamelCase      ,							hidden_size=3_7    )

       def 						UpperCAmelCase(   self      :     List[str]    ):
            self.config_tester.run_common_tests()

       def 						UpperCAmelCase(   self      :     int    ):
            a_    :     str										=    self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*__lowerCamelCase    )

       def 						UpperCAmelCase(   self      :     Optional[int]    ):
            a_    :     int										=    self.model_tester.prepare_config_and_inputs_for_decoder()
            self.model_tester.create_and_check_model_as_decoder(*__lowerCamelCase    )

       def 						UpperCAmelCase(   self      :     int    ):
            a_    :     Optional[Any]										=    self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*__lowerCamelCase    )

       def 						UpperCAmelCase(   self      :     Optional[Any]    ):
            a_    :     Dict										=    self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*__lowerCamelCase    )

       @slow
       def 						UpperCAmelCase(   self      :     str    ):
            for model_name in TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 a_    :     Tuple										=    TFEsmModel.from_pretrained(__lowerCamelCase    )
                 self.assertIsNotNone(__lowerCamelCase    )

       @unittest.skip("""Protein models do not support embedding resizing."""    )
       def 						UpperCAmelCase(   self      :     List[str]    ):
            pass

       @unittest.skip("""Protein models do not support embedding resizing."""    )
       def 						UpperCAmelCase(   self      :     List[Any]    ):
            pass

       def 						UpperCAmelCase(   self      :     List[Any]    ):
            a_    :     int										=    self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 a_    :     Union[str, Any]										=    model_class(__lowerCamelCase    )
                 assert isinstance(model.get_input_embeddings()      ,							tf.keras.layers.Layer    )
                 if model_class is TFEsmForMaskedLM:
                      # Output embedding test differs from the main test because they're a matrix, not a layer
                      a_    :     Union[str, Any]										=    model.get_bias()
                      assert isinstance(__lowerCamelCase      ,							__lowerCamelCase    )
                      for k, v in name.items():
                           assert isinstance(__lowerCamelCase      ,							tf.Variable    )
                 else:
                      a_    :     List[Any]										=    model.get_output_embeddings()
                      assert x is None
                      a_    :     List[Any]										=    model.get_bias()
                      assert name is None


@require_tf
class   a__	(						unittest.TestCase    ):

       @slow
       def 						UpperCAmelCase(   self      :     Union[str, Any]    ):
            a_    :     Any										=    TFEsmForMaskedLM.from_pretrained("""facebook/esm2_t6_8M_UR50D"""    )

            a_    :     List[str]										=    tf.constant([[0, 1, 2, 3, 4, 5]]    )
            a_    :     Any										=    model(__lowerCamelCase    )[0]
            a_    :     int										=    [1, 6, 3_3]
            self.assertEqual(list(output.numpy().shape    )      ,							__lowerCamelCase    )
            # compare the actual values for a slice.
            a_    :     Union[str, Any]										=    tf.constant(
                [
                    [
                        [8.9_2_1_5_1_8, -1_0.5_8_9_8_1_4, -6.4_6_7_1_3_0_7],
                        [-6.3_9_6_7_1_5_6, -1_3.9_1_1_3_7_7, -1.1_2_1_1_9_1_5],
                        [-7.7_8_1_2_4_7, -1_3.9_5_1_5_5_7, -3.7_4_0_5_9_2],
                    ]
                ]    )
            self.assertTrue(numpy.allclose(output[:, :3, :3].numpy()      ,							expected_slice.numpy()      ,							atol=1E-2    )    )

       @slow
       def 						UpperCAmelCase(   self      :     Optional[int]    ):
            a_    :     Optional[int]										=    TFEsmModel.from_pretrained("""facebook/esm2_t6_8M_UR50D"""    )

            a_    :     int										=    tf.constant([[0, 6, 4, 1_3, 5, 4, 1_6, 1_2, 1_1, 7, 2]]    )
            a_    :     Optional[Any]										=    model(__lowerCamelCase    )[0]
            # compare the actual values for a slice.
            a_    :     Dict										=    tf.constant(
                [
                    [
                        [0.1_4_4_4_3_0_9_2, 0.5_4_1_2_5_3_2_7, 0.3_2_4_7_7_3_9],
                        [0.3_0_3_4_0_4_8_4, 0.0_0_5_2_6_6_7_6, 0.3_1_0_7_7_7_2_2],
                        [0.3_2_2_7_8_0_4_3, -0.2_4_9_8_7_0_9_6, 0.3_4_1_4_6_2_8],
                    ]
                ]    )
            self.assertTrue(numpy.allclose(output[:, :3, :3].numpy()      ,							expected_slice.numpy()      ,							atol=1E-4    )    )








import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase        =     re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__lowerCamelCase        =     None



def  _a			(						):
     a_    :     Tuple										=    argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
     parser.add_argument("""data_file"""  ,  metavar="""data.json"""  ,  help="""Input data JSON file.""" )
     parser.add_argument("""pred_file"""  ,  metavar="""pred.json"""  ,  help="""Model predictions.""" )
     parser.add_argument(
         """--out-file"""  ,  """-o"""  ,  metavar="""eval.json"""  ,  help="""Write accuracy metrics to file (default is stdout).""" )
     parser.add_argument(
         """--na-prob-file"""  ,  """-n"""  ,  metavar="""na_prob.json"""  ,  help="""Model estimates of probability of no answer.""" )
     parser.add_argument(
         """--na-prob-thresh"""  ,  """-t"""  ,  type=__UpperCamelCase  ,  default=1.0  ,  help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""  ,  )
     parser.add_argument(
         """--out-image-dir"""  ,  """-p"""  ,  metavar="""out_images"""  ,  default=__UpperCamelCase  ,  help="""Save precision-recall curves to directory.""" )
     parser.add_argument("""--verbose"""  ,  """-v"""  ,  action="""store_true""" )
     if len(sys.argv ) == 1:
          parser.print_help()
          sys.exit(1 )
     return parser.parse_args()



def  _a			(						__UpperCamelCase ):
     a_    :     Union[str, Any]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     Union[str, Any]										=    bool(qa["""answers"""]["""text"""] )
     return qid_to_has_ans



def  _a			(						__UpperCamelCase ):

     def remove_articles(__UpperCamelCase ):
          return ARTICLES_REGEX.sub(""" """  ,  __UpperCamelCase )

     def white_space_fix(__UpperCamelCase ):
          return " ".join(text.split() )

     def remove_punc(__UpperCamelCase ):
          a_    :     List[str]										=    set(string.punctuation )
          return "".join(ch for ch in text if ch not in exclude )

     def lower(__UpperCamelCase ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase ) ) ) )



def  _a			(						__UpperCamelCase ):
     if not s:
          return []
     return normalize_answer(__UpperCamelCase ).split()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     return int(normalize_answer(__UpperCamelCase ) == normalize_answer(__UpperCamelCase ) )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     List[Any]										=    get_tokens(__UpperCamelCase )
     a_    :     str										=    get_tokens(__UpperCamelCase )
     a_    :     Tuple										=    collections.Counter(__UpperCamelCase ) & collections.Counter(__UpperCamelCase )
     a_    :     str										=    sum(common.values() )
     if len(__UpperCamelCase ) == 0 or len(__UpperCamelCase ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks )
     if num_same == 0:
          return 0
     a_    :     Any										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     Optional[int]										=    1.0 * num_same / len(__UpperCamelCase )
     a_    :     List[Any]										=    (2 * precision * recall) / (precision + recall)
     return fa



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Tuple										=    {}
     a_    :     Optional[int]										=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_    :     str										=    qa["""id"""]
                    a_    :     Dict										=    [t for t in qa["""answers"""]["""text"""] if normalize_answer(__UpperCamelCase )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         a_    :     Union[str, Any]										=    [""""""]
                    if qid not in preds:
                         print(F'''Missing prediction for {qid}''' )
                         continue
                    a_    :     int										=    preds[qid]
                    # Take max over all gold answers
                    a_    :     Union[str, Any]										=    max(compute_exact(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
                    a_    :     int										=    max(compute_fa(__UpperCamelCase  ,  __UpperCamelCase ) for a in gold_answers )
     return exact_scores, fa_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[Any]										=    {}
     for qid, s in scores.items():
          a_    :     Optional[int]										=    na_probs[qid] > na_prob_thresh
          if pred_na:
               a_    :     List[Any]										=    float(not qid_to_has_ans[qid] )
          else:
               a_    :     List[Any]										=    s
     return new_scores



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None ):
     if not qid_list:
          a_    :     List[Any]										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores.values() ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores.values() ) / total),
                  ("""total""", total),
              ] )
     else:
          a_    :     Tuple										=    len(__UpperCamelCase )
          return collections.OrderedDict(
              [
                  ("""exact""", 1_00.0 * sum(exact_scores[k] for k in qid_list ) / total),
                  ("""f1""", 1_00.0 * sum(fa_scores[k] for k in qid_list ) / total),
                  ("""total""", total),
              ] )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     for k in new_eval:
          a_    :     Any										=    new_eval[k]



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     plt.step(__UpperCamelCase  ,  __UpperCamelCase  ,  color="""b"""  ,  alpha=0.2  ,  where="""post""" )
     plt.fill_between(__UpperCamelCase  ,  __UpperCamelCase  ,  step="""post"""  ,  alpha=0.2  ,  color="""b""" )
     plt.xlabel("""Recall""" )
     plt.ylabel("""Precision""" )
     plt.xlim([0.0, 1.05] )
     plt.ylim([0.0, 1.05] )
     plt.title(__UpperCamelCase )
     plt.savefig(__UpperCamelCase )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase=None  ,  __UpperCamelCase=None ):
     a_    :     List[Any]										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     a_    :     int										=    0.0
     a_    :     Tuple										=    1.0
     a_    :     Union[str, Any]										=    0.0
     a_    :     List[str]										=    [1.0]
     a_    :     int										=    [0.0]
     a_    :     Optional[int]										=    0.0
     for i, qid in enumerate(__UpperCamelCase ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          a_    :     Union[str, Any]										=    true_pos / float(i + 1 )
          a_    :     Tuple										=    true_pos / float(__UpperCamelCase )
          if i == len(__UpperCamelCase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(__UpperCamelCase )
               recalls.append(__UpperCamelCase )
     if out_image:
          plot_pr_curve(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     return {"ap": 1_00.0 * avg_prec}



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if out_image_dir and not os.path.exists(__UpperCamelCase ):
          os.makedirs(__UpperCamelCase )
     a_    :     Any										=    sum(1 for v in qid_to_has_ans.values() if v )
     if num_true_pos == 0:
          return
     a_    :     Union[str, Any]										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_exact.png""" )  ,  title="""Precision-Recall curve for Exact Match score"""  ,  )
     a_    :     Any										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_f1.png""" )  ,  title="""Precision-Recall curve for F1 score"""  ,  )
     a_    :     int										=    {k: float(__UpperCamelCase ) for k, v in qid_to_has_ans.items()}
     a_    :     Dict										=    make_precision_recall_eval(
         __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  out_image=os.path.join(__UpperCamelCase  ,  """pr_oracle.png""" )  ,  title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""  ,  )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_exact""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_f1""" )
     merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """pr_oracle""" )



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     if not qid_list:
          return
     a_    :     List[Any]										=    [na_probs[k] for k in qid_list]
     a_    :     List[Any]										=    np.ones_like(__UpperCamelCase ) / float(len(__UpperCamelCase ) )
     plt.hist(__UpperCamelCase  ,  weights=__UpperCamelCase  ,  bins=2_0  ,  range=(0.0, 1.0) )
     plt.xlabel("""Model probability of no-answer""" )
     plt.ylabel("""Proportion of dataset""" )
     plt.title(F'''Histogram of no-answer probability: {name}''' )
     plt.savefig(os.path.join(__UpperCamelCase  ,  F'''na_prob_hist_{name}.png''' ) )
     plt.clf()



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_    :     Optional[int]										=    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
     a_    :     Tuple										=    num_no_ans
     a_    :     List[Any]										=    cur_score
     a_    :     List[Any]										=    0.0
     a_    :     int										=    sorted(__UpperCamelCase  ,  key=lambda __UpperCamelCase : na_probs[k] )
     for i, qid in enumerate(__UpperCamelCase ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               a_    :     Optional[Any]										=    scores[qid]
          else:
               if preds[qid]:
                    a_    :     Optional[int]										=    -1
               else:
                    a_    :     Optional[int]										=    0
          cur_score += diff
          if cur_score > best_score:
               a_    :     str										=    cur_score
               a_    :     int										=    na_probs[qid]
     return 1_00.0 * best_score / len(__UpperCamelCase ), best_thresh



def  _a			(						__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase ):
     a_			,			a_    :     Optional[int]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_			,			a_    :     Optional[Any]										=    find_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     a_    :     Dict										=    best_exact
     a_    :     Optional[Any]										=    exact_thresh
     a_    :     Optional[Any]										=    best_fa
     a_    :     Union[str, Any]										=    fa_thresh



def  _a			(						):
     with open(OPTS.data_file ) as f:
          a_    :     str										=    json.load(__UpperCamelCase )
          a_    :     Optional[int]										=    dataset_json["""data"""]
     with open(OPTS.pred_file ) as f:
          a_    :     Dict										=    json.load(__UpperCamelCase )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file ) as f:
               a_    :     List[str]										=    json.load(__UpperCamelCase )
     else:
          a_    :     List[str]										=    {k: 0.0 for k in preds}
     a_    :     Any										=    make_qid_to_has_ans(__UpperCamelCase )  # maps qid to True/False
     a_    :     Any										=    [k for k, v in qid_to_has_ans.items() if v]
     a_    :     Union[str, Any]										=    [k for k, v in qid_to_has_ans.items() if not v]
     a_			,			a_    :     List[str]										=    get_raw_scores(__UpperCamelCase  ,  __UpperCamelCase )
     a_    :     int										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     Optional[int]										=    apply_no_ans_threshold(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.na_prob_thresh )
     a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase )
     if has_ans_qids:
          a_    :     Dict										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """HasAns""" )
     if no_ans_qids:
          a_    :     List[str]										=    make_eval_dict(__UpperCamelCase  ,  __UpperCamelCase  ,  qid_list=__UpperCamelCase )
          merge_eval(__UpperCamelCase  ,  __UpperCamelCase  ,  """NoAns""" )
     if OPTS.na_prob_file:
          find_all_best_thresh(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(__UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """hasAns""" )
          histogram_na_prob(__UpperCamelCase  ,  __UpperCamelCase  ,  OPTS.out_image_dir  ,  """noAns""" )
     if OPTS.out_file:
          with open(OPTS.out_file  ,  """w""" ) as f:
               json.dump(__UpperCamelCase  ,  __UpperCamelCase )
     else:
          print(json.dumps(__UpperCamelCase  ,  indent=2 ) )


if __name__ == "__main__":
       __lowerCamelCase        =     parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('''Agg''')
              import matplotlib.pyplot as plt
       main()



