






import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


__lowerCamelCase	:		List[Any]										=   {
    """return_dict""": False,
    """output_hidden_states""": True,
    """output_attentions""": True,
    """torchscript""": True,
    """torch_dtype""": """float16""",
    """use_bfloat16""": True,
    """tf_legacy_loss""": True,
    """pruned_heads""": {"""a""": 1},
    """tie_word_embeddings""": False,
    """is_decoder""": True,
    """cross_attention_hidden_size""": 1_28,
    """add_cross_attention""": True,
    """tie_encoder_decoder""": True,
    """max_length""": 50,
    """min_length""": 3,
    """do_sample""": True,
    """early_stopping""": True,
    """num_beams""": 3,
    """num_beam_groups""": 3,
    """diversity_penalty""": 0.5,
    """temperature""": 2.0,
    """top_k""": 10,
    """top_p""": 0.7,
    """typical_p""": 0.2,
    """repetition_penalty""": 0.8,
    """length_penalty""": 0.8,
    """no_repeat_ngram_size""": 5,
    """encoder_no_repeat_ngram_size""": 5,
    """bad_words_ids""": [1, 2, 3],
    """num_return_sequences""": 3,
    """chunk_size_feed_forward""": 5,
    """output_scores""": True,
    """return_dict_in_generate""": True,
    """forced_bos_token_id""": 2,
    """forced_eos_token_id""": 3,
    """remove_invalid_values""": True,
    """architectures""": ["""BertModel"""],
    """finetuning_task""": """translation""",
    """id2label""": {0: """label"""},
    """label2id""": {"""label""": """0"""},
    """tokenizer_class""": """BertTokenizerFast""",
    """prefix""": """prefix""",
    """bos_token_id""": 6,
    """pad_token_id""": 7,
    """eos_token_id""": 8,
    """sep_token_id""": 9,
    """decoder_start_token_id""": 10,
    """exponential_decay_length_penalty""": (5, 1.01),
    """suppress_tokens""": [0, 1],
    """begin_suppress_tokens""": 2,
    """task_specific_params""": {"""translation""": """some_params"""},
    """problem_type""": """regression""",
}







@is_staging_test
class  _lowercase   (   unittest.TestCase						):
      @classmethod
      def 						lowercase__					(     cls   ):
       snake_case__  :    Optional[int]							       =TOKEN
       HfFolder.save_token(a   )
      @classmethod
      def 						lowercase__					(     cls   ):
       try:
        delete_repo(token=cls._token				,			repo_id="""test-config"""   )
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token				,			repo_id="""valid_org/test-config-org"""   )
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token				,			repo_id="""test-dynamic-config"""   )
       except HTTPError:
        pass
      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[Any]							       =BertConfig(
           vocab_size=9_9				,			hidden_size=3_2				,			num_hidden_layers=5				,			num_attention_heads=4				,			intermediate_size=3_7   )
       config.push_to_hub("""test-config"""				,			use_auth_token=self._token   )

       snake_case__  :    Tuple							       =BertConfig.from_pretrained(F"{USER}/test-config"   )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(a				,			getattr(a				,			a   )   )

        # Reset repo
       delete_repo(token=self._token				,			repo_id="""test-config"""   )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        config.save_pretrained(a				,			repo_id="""test-config"""				,			push_to_hub=a				,			use_auth_token=self._token   )

       snake_case__  :    Union[str, Any]							       =BertConfig.from_pretrained(F"{USER}/test-config"   )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(a				,			getattr(a				,			a   )   )
      def 						lowercase__					(     self   ):
       snake_case__  :    Optional[Any]							       =BertConfig(
           vocab_size=9_9				,			hidden_size=3_2				,			num_hidden_layers=5				,			num_attention_heads=4				,			intermediate_size=3_7   )
       config.push_to_hub("""valid_org/test-config-org"""				,			use_auth_token=self._token   )

       snake_case__  :    Tuple							       =BertConfig.from_pretrained("""valid_org/test-config-org"""   )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(a				,			getattr(a				,			a   )   )

        # Reset repo
       delete_repo(token=self._token				,			repo_id="""valid_org/test-config-org"""   )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        config.save_pretrained(
            a				,			repo_id="""valid_org/test-config-org"""				,			push_to_hub=a				,			use_auth_token=self._token   )

       snake_case__  :    Optional[int]							       =BertConfig.from_pretrained("""valid_org/test-config-org"""   )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(a				,			getattr(a				,			a   )   )



      def 						lowercase__					(     self   ):
       CustomConfig.register_for_auto_class()
       snake_case__  :    int							       =CustomConfig(attribute=4_2   )

       config.push_to_hub("""test-dynamic-config"""				,			use_auth_token=self._token   )

       # This has added the proper auto_map field to the config
       self.assertDictEqual(config.auto_map				,			{"""AutoConfig""": """custom_configuration.CustomConfig"""}   )

       snake_case__  :    Any							       =AutoConfig.from_pretrained(F"{USER}/test-dynamic-config"				,			trust_remote_code=a   )
       # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
       self.assertEqual(new_config.__class__.__name__				,			"""CustomConfig"""   )
       self.assertEqual(new_config.attribute				,			4_2   )







class  _lowercase   (   unittest.TestCase						):
      def 						lowercase__					(     self   ):
       snake_case__  :    int							       =GPTaConfig()

       # attempt to modify each of int/float/bool/str config records and verify they were updated
       snake_case__  :    List[Any]							       =c.n_embd + 1  # int
       snake_case__  :    int							       =c.resid_pdrop + 1.0  # float
       snake_case__  :    List[Any]							       =not c.scale_attn_weights  # bool
       snake_case__  :    int							       =c.summary_type + """foo"""  # str
       c.update_from_string(
           F"n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"   )
       self.assertEqual(a				,			c.n_embd				,			"""mismatch for key: n_embd"""   )
       self.assertEqual(a				,			c.resid_pdrop				,			"""mismatch for key: resid_pdrop"""   )
       self.assertEqual(a				,			c.scale_attn_weights				,			"""mismatch for key: scale_attn_weights"""   )
       self.assertEqual(a				,			c.summary_type				,			"""mismatch for key: summary_type"""   )
      def 						lowercase__					(     self   ):
       snake_case__  :    str							       =PretrainedConfig()
       snake_case__  :    List[Any]							       =[key for key in base_config.__dict__ if key not in config_common_kwargs]
       # If this part of the test fails, you have arguments to addin config_common_kwargs above.
       self.assertListEqual(
           a				,			["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]   )
       snake_case__  :    Optional[Any]							       =[key for key, value in config_common_kwargs.items() if value == getattr(a				,			a   )]
       if len(a   ) > 0:
        raise ValueError(
            """The following keys are set with the default values in"""
            """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
            F" {', '.join(a   )}."   )
      def 						lowercase__					(     self   ):
       with self.assertRaises(a   ):
        # config is in subfolder, the following should not work without specifying the subfolder
        snake_case__  :    Any							       =BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""   )

       snake_case__  :    Union[str, Any]							       =BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""				,			subfolder="""bert"""   )

       self.assertIsNotNone(a   )
      def 						lowercase__					(     self   ):
       # A mock response for an HTTP head request to emulate server down
       snake_case__  :    str							       =mock.Mock()
       snake_case__  :    Dict							       =5_0_0
       snake_case__  :    int							       ={}
       snake_case__  :    str							       =HTTPError
       snake_case__  :    Optional[int]							       ={}

       # Download this model to make sure it's in the cache.
       snake_case__  :    Optional[Any]							       =BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )

       # Under the mock environment we get a 500 error when trying to reach the model.
       with mock.patch("""requests.Session.request"""				,			return_value=a   ) as mock_head:
        snake_case__  :    Optional[int]							       =BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )
        # This check we did call the fake head request
        mock_head.assert_called()
      def 						lowercase__					(     self   ):
       # This test is for deprecated behavior and can be removed in v5
       snake_case__  :    Dict							       =BertConfig.from_pretrained(
           """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""   )
      def 						lowercase__					(     self   ):
       snake_case__  :    Dict							       =AutoConfig.from_pretrained("""bert-base-cased"""   )
       snake_case__  :    Dict							       =["""config.4.0.0.json"""]

       with tempfile.TemporaryDirectory() as tmp_dir:
        configuration.save_pretrained(a   )
        snake_case__  :    Union[str, Any]							       =2
        json.dump(configuration.to_dict()				,			open(os.path.join(a				,			"""config.4.0.0.json"""   )				,			"""w"""   )   )

        # This should pick the new configuration file as the version of Transformers is > 4.0.0
        snake_case__  :    List[str]							       =AutoConfig.from_pretrained(a   )
        self.assertEqual(new_configuration.hidden_size				,			2   )

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old configuration file as the version of Transformers is < 4.42.0
        snake_case__  :    int							       =["""config.42.0.0.json"""]
        snake_case__  :    Union[str, Any]							       =7_6_8
        configuration.save_pretrained(a   )
        shutil.move(os.path.join(a				,			"""config.4.0.0.json"""   )				,			os.path.join(a				,			"""config.42.0.0.json"""   )   )
        snake_case__  :    int							       =AutoConfig.from_pretrained(a   )
        self.assertEqual(new_configuration.hidden_size				,			7_6_8   )



      def 						lowercase__					(     self   ):
       # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
       snake_case__  :    List[Any]							       ="""hf-internal-testing/test-two-configs"""

       import transformers as new_transformers

       snake_case__  :    Any							       ="""v4.0.0"""
       snake_case__  :    Any							       =new_transformers.models.auto.AutoConfig.from_pretrained(
           a				,			return_unused_kwargs=a   )
       self.assertEqual(new_configuration.hidden_size				,			2   )
       # This checks `_configuration_file` ia not kept in the kwargs by mistake.
       self.assertDictEqual(a				,			{}   )

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       snake_case__  :    int							       ="""v3.0.0"""
       snake_case__  :    Dict							       =old_transformers.models.auto.AutoConfig.from_pretrained(a   )
       self.assertEqual(old_configuration.hidden_size				,			7_6_8   )













from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


__lowerCamelCase	:		Any										=   """\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
"""

__lowerCamelCase	:		str										=   """\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
"""

__lowerCamelCase	:		str										=   """
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    \"accuracy\": Accuracy
    \"f1\": F1 score
    \"pearson\": Pearson Correlation
    \"spearmanr\": Spearman Correlation
    \"matthews_correlation\": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'stsb')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})
    {'pearson': 1.0, 'spearmanr': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'cola')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""







def 			A__							(      _a  :					int		,  _a  :					str	):





 '''simple docstring'''







 return float((preds == labels).mean()	)







def 			A__							(      _a  :					Optional[Any]		,  _a  :					Any	):





 '''simple docstring'''







 snake_case__  :    str							       =simple_accuracy(_a		,  _a	)
 snake_case__  :    str							       =float(fa_score(y_true=_a		,  y_pred=_a	)	)
 return {
     "accuracy": acc,
     "f1": fa,
 }







def 			A__							(      _a  :					int		,  _a  :					Optional[int]	):





 '''simple docstring'''







 snake_case__  :    str							       =float(pearsonr(_a		,  _a	)[0]	)
 snake_case__  :    Optional[int]							       =float(spearmanr(_a		,  _a	)[0]	)
 return {
     "pearson": pearson_corr,
     "spearmanr": spearman_corr,
 }







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,				_KWARGS_DESCRIPTION						)
class  _lowercase   (   datasets.Metric						):
      def 						lowercase__					(     self   ):
       if self.config_name not in [
           "sst2",
           "mnli",
           "mnli_mismatched",
           "mnli_matched",
           "cola",
           "stsb",
           "mrpc",
           "qqp",
           "qnli",
           "rte",
           "wnli",
           "hans",
       ]:
        raise KeyError(
            """You should supply a configuration name selected in """
            """[\"sst2\", \"mnli\", \"mnli_mismatched\", \"mnli_matched\", """
            """\"cola\", \"stsb\", \"mrpc\", \"qqp\", \"qnli\", \"rte\", \"wnli\", \"hans\"]"""   )
       return datasets.MetricInfo(
           description=_DESCRIPTION				,			citation=_CITATION				,			inputs_description=_KWARGS_DESCRIPTION				,			features=datasets.Features(
               {
                   """predictions""": datasets.Value("""int64""" if self.config_name != """stsb""" else """float32"""   ),
                   """references""": datasets.Value("""int64""" if self.config_name != """stsb""" else """float32"""   ),
               }   )				,			codebase_urls=[]				,			reference_urls=[]				,			format="""numpy"""				,			)



      def 						lowercase__					(     self				,			a				,			a   ):
       if self.config_name == "cola":
        return {"matthews_correlation": matthews_corrcoef(a				,			a   )}
       elif self.config_name == "stsb":
        return pearson_and_spearman(a				,			a   )
       elif self.config_name in ["mrpc", "qqp"]:
        return acc_and_fa(a				,			a   )
       elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
        return {"accuracy": simple_accuracy(a				,			a   )}
       else:
        raise KeyError(
            """You should supply a configuration name selected in """
            """[\"sst2\", \"mnli\", \"mnli_mismatched\", \"mnli_matched\", """
            """\"cola\", \"stsb\", \"mrpc\", \"qqp\", \"qnli\", \"rte\", \"wnli\", \"hans\"]"""   )





