We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[example/glue] run_glue compute metrics fail for bart like models
This PR #7126 introduced multiple predictions for trainer. This breaks the `compute_metrics_fn` of `run_glue.py`  for `bart` like models which return multiple predictions.

For `BartForSequenceClassfication` `p.predictions`  is a `tuple`, so following code fails
https://github.com/huggingface/transformers/blob/1d90d0f386af2af52017d51c421e71a51ec94de0/examples/text-classification/run_glue.py#L154

@sgugger 


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/text-classification/run_glue.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
3 # Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 """ Finetuning the library models for sequence classification on GLUE."""
17 
18 
19 import dataclasses
20 import logging
21 import os
22 import sys
23 from dataclasses import dataclass, field
24 from typing import Callable, Dict, Optional
25 
26 import numpy as np
27 
28 from transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, EvalPrediction, GlueDataset
29 from transformers import GlueDataTrainingArguments as DataTrainingArguments
30 from transformers import (
31     HfArgumentParser,
32     Trainer,
33     TrainingArguments,
34     glue_compute_metrics,
35     glue_output_modes,
36     glue_tasks_num_labels,
37     set_seed,
38 )
39 
40 
41 logger = logging.getLogger(__name__)
42 
43 
44 @dataclass
45 class ModelArguments:
46     """
47     Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
48     """
49 
50     model_name_or_path: str = field(
51         metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
52     )
53     config_name: Optional[str] = field(
54         default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
55     )
56     tokenizer_name: Optional[str] = field(
57         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
58     )
59     cache_dir: Optional[str] = field(
60         default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
61     )
62 
63 
64 def main():
65     # See all possible arguments in src/transformers/training_args.py
66     # or by passing the --help flag to this script.
67     # We now keep distinct sets of args, for a cleaner separation of concerns.
68 
69     parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
70 
71     if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
72         # If we pass only one argument to the script and it's the path to a json file,
73         # let's parse it to get our arguments.
74         model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
75     else:
76         model_args, data_args, training_args = parser.parse_args_into_dataclasses()
77 
78     if (
79         os.path.exists(training_args.output_dir)
80         and os.listdir(training_args.output_dir)
81         and training_args.do_train
82         and not training_args.overwrite_output_dir
83     ):
84         raise ValueError(
85             f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
86         )
87 
88     # Setup logging
89     logging.basicConfig(
90         format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
91         datefmt="%m/%d/%Y %H:%M:%S",
92         level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
93     )
94     logger.warning(
95         "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
96         training_args.local_rank,
97         training_args.device,
98         training_args.n_gpu,
99         bool(training_args.local_rank != -1),
100         training_args.fp16,
101     )
102     logger.info("Training/evaluation parameters %s", training_args)
103 
104     # Set seed
105     set_seed(training_args.seed)
106 
107     try:
108         num_labels = glue_tasks_num_labels[data_args.task_name]
109         output_mode = glue_output_modes[data_args.task_name]
110     except KeyError:
111         raise ValueError("Task not found: %s" % (data_args.task_name))
112 
113     # Load pretrained model and tokenizer
114     #
115     # Distributed training:
116     # The .from_pretrained methods guarantee that only one local process can concurrently
117     # download model & vocab.
118 
119     config = AutoConfig.from_pretrained(
120         model_args.config_name if model_args.config_name else model_args.model_name_or_path,
121         num_labels=num_labels,
122         finetuning_task=data_args.task_name,
123         cache_dir=model_args.cache_dir,
124     )
125     tokenizer = AutoTokenizer.from_pretrained(
126         model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
127         cache_dir=model_args.cache_dir,
128     )
129     model = AutoModelForSequenceClassification.from_pretrained(
130         model_args.model_name_or_path,
131         from_tf=bool(".ckpt" in model_args.model_name_or_path),
132         config=config,
133         cache_dir=model_args.cache_dir,
134     )
135 
136     # Get datasets
137     train_dataset = (
138         GlueDataset(data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir) if training_args.do_train else None
139     )
140     eval_dataset = (
141         GlueDataset(data_args, tokenizer=tokenizer, mode="dev", cache_dir=model_args.cache_dir)
142         if training_args.do_eval
143         else None
144     )
145     test_dataset = (
146         GlueDataset(data_args, tokenizer=tokenizer, mode="test", cache_dir=model_args.cache_dir)
147         if training_args.do_predict
148         else None
149     )
150 
151     def build_compute_metrics_fn(task_name: str) -> Callable[[EvalPrediction], Dict]:
152         def compute_metrics_fn(p: EvalPrediction):
153             if output_mode == "classification":
154                 preds = np.argmax(p.predictions, axis=1)
155             elif output_mode == "regression":
156                 preds = np.squeeze(p.predictions)
157             return glue_compute_metrics(task_name, preds, p.label_ids)
158 
159         return compute_metrics_fn
160 
161     # Initialize our Trainer
162     trainer = Trainer(
163         model=model,
164         args=training_args,
165         train_dataset=train_dataset,
166         eval_dataset=eval_dataset,
167         compute_metrics=build_compute_metrics_fn(data_args.task_name),
168     )
169 
170     # Training
171     if training_args.do_train:
172         trainer.train(
173             model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
174         )
175         trainer.save_model()
176         # For convenience, we also re-save the tokenizer to the same directory,
177         # so that you can share your model easily on huggingface.co/models =)
178         if trainer.is_world_master():
179             tokenizer.save_pretrained(training_args.output_dir)
180 
181     # Evaluation
182     eval_results = {}
183     if training_args.do_eval:
184         logger.info("*** Evaluate ***")
185 
186         # Loop to handle MNLI double evaluation (matched, mis-matched)
187         eval_datasets = [eval_dataset]
188         if data_args.task_name == "mnli":
189             mnli_mm_data_args = dataclasses.replace(data_args, task_name="mnli-mm")
190             eval_datasets.append(
191                 GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, mode="dev", cache_dir=model_args.cache_dir)
192             )
193 
194         for eval_dataset in eval_datasets:
195             trainer.compute_metrics = build_compute_metrics_fn(eval_dataset.args.task_name)
196             eval_result = trainer.evaluate(eval_dataset=eval_dataset)
197 
198             output_eval_file = os.path.join(
199                 training_args.output_dir, f"eval_results_{eval_dataset.args.task_name}.txt"
200             )
201             if trainer.is_world_master():
202                 with open(output_eval_file, "w") as writer:
203                     logger.info("***** Eval results {} *****".format(eval_dataset.args.task_name))
204                     for key, value in eval_result.items():
205                         logger.info("  %s = %s", key, value)
206                         writer.write("%s = %s\n" % (key, value))
207 
208             eval_results.update(eval_result)
209 
210     if training_args.do_predict:
211         logging.info("*** Test ***")
212         test_datasets = [test_dataset]
213         if data_args.task_name == "mnli":
214             mnli_mm_data_args = dataclasses.replace(data_args, task_name="mnli-mm")
215             test_datasets.append(
216                 GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, mode="test", cache_dir=model_args.cache_dir)
217             )
218 
219         for test_dataset in test_datasets:
220             predictions = trainer.predict(test_dataset=test_dataset).predictions
221             if output_mode == "classification":
222                 predictions = np.argmax(predictions, axis=1)
223 
224             output_test_file = os.path.join(
225                 training_args.output_dir, f"test_results_{test_dataset.args.task_name}.txt"
226             )
227             if trainer.is_world_master():
228                 with open(output_test_file, "w") as writer:
229                     logger.info("***** Test results {} *****".format(test_dataset.args.task_name))
230                     writer.write("index\tprediction\n")
231                     for index, item in enumerate(predictions):
232                         if output_mode == "regression":
233                             writer.write("%d\t%3.3f\n" % (index, item))
234                         else:
235                             item = test_dataset.get_labels()[item]
236                             writer.write("%d\t%s\n" % (index, item))
237     return eval_results
238 
239 
240 def _mp_fn(index):
241     # For xla_spawn (TPUs)
242     main()
243 
244 
245 if __name__ == "__main__":
246     main()
247
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/text-classification/run_glue.py b/examples/text-classification/run_glue.py
--- a/examples/text-classification/run_glue.py
+++ b/examples/text-classification/run_glue.py
@@ -150,10 +150,11 @@
 
     def build_compute_metrics_fn(task_name: str) -> Callable[[EvalPrediction], Dict]:
         def compute_metrics_fn(p: EvalPrediction):
+            preds = p.predictions[0] if type(p.predictions) == tuple else p.predictions
             if output_mode == "classification":
-                preds = np.argmax(p.predictions, axis=1)
-            elif output_mode == "regression":
-                preds = np.squeeze(p.predictions)
+                preds = np.argmax(preds, axis=1)
+            else:  # regression
+                preds = np.squeeze(preds)
             return glue_compute_metrics(task_name, preds, p.label_ids)
 
         return compute_metrics_fn

{"golden_diff": "diff --git a/examples/text-classification/run_glue.py b/examples/text-classification/run_glue.py\n--- a/examples/text-classification/run_glue.py\n+++ b/examples/text-classification/run_glue.py\n@@ -150,10 +150,11 @@\n \n     def build_compute_metrics_fn(task_name: str) -> Callable[[EvalPrediction], Dict]:\n         def compute_metrics_fn(p: EvalPrediction):\n+            preds = p.predictions[0] if type(p.predictions) == tuple else p.predictions\n             if output_mode == \"classification\":\n-                preds = np.argmax(p.predictions, axis=1)\n-            elif output_mode == \"regression\":\n-                preds = np.squeeze(p.predictions)\n+                preds = np.argmax(preds, axis=1)\n+            else:  # regression\n+                preds = np.squeeze(preds)\n             return glue_compute_metrics(task_name, preds, p.label_ids)\n \n         return compute_metrics_fn\n", "issue": "[example/glue] run_glue compute metrics fail for bart like models\nThis PR #7126 introduced multiple predictions for trainer. This breaks the `compute_metrics_fn` of `run_glue.py`  for `bart` like models which return multiple predictions.\r\n\r\nFor `BartForSequenceClassfication` `p.predictions`  is a `tuple`, so following code fails\r\nhttps://github.com/huggingface/transformers/blob/1d90d0f386af2af52017d51c421e71a51ec94de0/examples/text-classification/run_glue.py#L154\r\n\r\n@sgugger \r\n\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\" Finetuning the library models for sequence classification on GLUE.\"\"\"\n\n\nimport dataclasses\nimport logging\nimport os\nimport sys\nfrom dataclasses import dataclass, field\nfrom typing import Callable, Dict, Optional\n\nimport numpy as np\n\nfrom transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, EvalPrediction, GlueDataset\nfrom transformers import GlueDataTrainingArguments as DataTrainingArguments\nfrom transformers import (\n    HfArgumentParser,\n    Trainer,\n    TrainingArguments,\n    glue_compute_metrics,\n    glue_output_modes,\n    glue_tasks_num_labels,\n    set_seed,\n)\n\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclass\nclass ModelArguments:\n    \"\"\"\n    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n    \"\"\"\n\n    model_name_or_path: str = field(\n        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n    )\n    config_name: Optional[str] = field(\n        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n    )\n    tokenizer_name: Optional[str] = field(\n        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n    )\n    cache_dir: Optional[str] = field(\n        default=None, metadata={\"help\": \"Where do you want to store the pretrained models downloaded from s3\"}\n    )\n\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    if (\n        os.path.exists(training_args.output_dir)\n        and os.listdir(training_args.output_dir)\n        and training_args.do_train\n        and not training_args.overwrite_output_dir\n    ):\n        raise ValueError(\n            f\"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome.\"\n        )\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,\n    )\n    logger.warning(\n        \"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\",\n        training_args.local_rank,\n        training_args.device,\n        training_args.n_gpu,\n        bool(training_args.local_rank != -1),\n        training_args.fp16,\n    )\n    logger.info(\"Training/evaluation parameters %s\", training_args)\n\n    # Set seed\n    set_seed(training_args.seed)\n\n    try:\n        num_labels = glue_tasks_num_labels[data_args.task_name]\n        output_mode = glue_output_modes[data_args.task_name]\n    except KeyError:\n        raise ValueError(\"Task not found: %s\" % (data_args.task_name))\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        num_labels=num_labels,\n        finetuning_task=data_args.task_name,\n        cache_dir=model_args.cache_dir,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n    )\n    model = AutoModelForSequenceClassification.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n    )\n\n    # Get datasets\n    train_dataset = (\n        GlueDataset(data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir) if training_args.do_train else None\n    )\n    eval_dataset = (\n        GlueDataset(data_args, tokenizer=tokenizer, mode=\"dev\", cache_dir=model_args.cache_dir)\n        if training_args.do_eval\n        else None\n    )\n    test_dataset = (\n        GlueDataset(data_args, tokenizer=tokenizer, mode=\"test\", cache_dir=model_args.cache_dir)\n        if training_args.do_predict\n        else None\n    )\n\n    def build_compute_metrics_fn(task_name: str) -> Callable[[EvalPrediction], Dict]:\n        def compute_metrics_fn(p: EvalPrediction):\n            if output_mode == \"classification\":\n                preds = np.argmax(p.predictions, axis=1)\n            elif output_mode == \"regression\":\n                preds = np.squeeze(p.predictions)\n            return glue_compute_metrics(task_name, preds, p.label_ids)\n\n        return compute_metrics_fn\n\n    # Initialize our Trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset,\n        eval_dataset=eval_dataset,\n        compute_metrics=build_compute_metrics_fn(data_args.task_name),\n    )\n\n    # Training\n    if training_args.do_train:\n        trainer.train(\n            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None\n        )\n        trainer.save_model()\n        # For convenience, we also re-save the tokenizer to the same directory,\n        # so that you can share your model easily on huggingface.co/models =)\n        if trainer.is_world_master():\n            tokenizer.save_pretrained(training_args.output_dir)\n\n    # Evaluation\n    eval_results = {}\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n\n        # Loop to handle MNLI double evaluation (matched, mis-matched)\n        eval_datasets = [eval_dataset]\n        if data_args.task_name == \"mnli\":\n            mnli_mm_data_args = dataclasses.replace(data_args, task_name=\"mnli-mm\")\n            eval_datasets.append(\n                GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, mode=\"dev\", cache_dir=model_args.cache_dir)\n            )\n\n        for eval_dataset in eval_datasets:\n            trainer.compute_metrics = build_compute_metrics_fn(eval_dataset.args.task_name)\n            eval_result = trainer.evaluate(eval_dataset=eval_dataset)\n\n            output_eval_file = os.path.join(\n                training_args.output_dir, f\"eval_results_{eval_dataset.args.task_name}.txt\"\n            )\n            if trainer.is_world_master():\n                with open(output_eval_file, \"w\") as writer:\n                    logger.info(\"***** Eval results {} *****\".format(eval_dataset.args.task_name))\n                    for key, value in eval_result.items():\n                        logger.info(\"  %s = %s\", key, value)\n                        writer.write(\"%s = %s\\n\" % (key, value))\n\n            eval_results.update(eval_result)\n\n    if training_args.do_predict:\n        logging.info(\"*** Test ***\")\n        test_datasets = [test_dataset]\n        if data_args.task_name == \"mnli\":\n            mnli_mm_data_args = dataclasses.replace(data_args, task_name=\"mnli-mm\")\n            test_datasets.append(\n                GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, mode=\"test\", cache_dir=model_args.cache_dir)\n            )\n\n        for test_dataset in test_datasets:\n            predictions = trainer.predict(test_dataset=test_dataset).predictions\n            if output_mode == \"classification\":\n                predictions = np.argmax(predictions, axis=1)\n\n            output_test_file = os.path.join(\n                training_args.output_dir, f\"test_results_{test_dataset.args.task_name}.txt\"\n            )\n            if trainer.is_world_master():\n                with open(output_test_file, \"w\") as writer:\n                    logger.info(\"***** Test results {} *****\".format(test_dataset.args.task_name))\n                    writer.write(\"index\\tprediction\\n\")\n                    for index, item in enumerate(predictions):\n                        if output_mode == \"regression\":\n                            writer.write(\"%d\\t%3.3f\\n\" % (index, item))\n                        else:\n                            item = test_dataset.get_labels()[item]\n                            writer.write(\"%d\\t%s\\n\" % (index, item))\n    return eval_results\n\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n    main()\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "examples/text-classification/run_glue.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\" Finetuning the library models for sequence classification on GLUE.\"\"\"\n\n\nimport dataclasses\nimport logging\nimport os\nimport sys\nfrom dataclasses import dataclass, field\nfrom typing import Callable, Dict, Optional\n\nimport numpy as np\n\nfrom transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, EvalPrediction, GlueDataset\nfrom transformers import GlueDataTrainingArguments as DataTrainingArguments\nfrom transformers import (\n    HfArgumentParser,\n    Trainer,\n    TrainingArguments,\n    glue_compute_metrics,\n    glue_output_modes,\n    glue_tasks_num_labels,\n    set_seed,\n)\n\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclass\nclass ModelArguments:\n    \"\"\"\n    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n    \"\"\"\n\n    model_name_or_path: str = field(\n        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n    )\n    config_name: Optional[str] = field(\n        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n    )\n    tokenizer_name: Optional[str] = field(\n        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n    )\n    cache_dir: Optional[str] = field(\n        default=None, metadata={\"help\": \"Where do you want to store the pretrained models downloaded from s3\"}\n    )\n\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    if (\n        os.path.exists(training_args.output_dir)\n        and os.listdir(training_args.output_dir)\n        and training_args.do_train\n        and not training_args.overwrite_output_dir\n    ):\n        raise ValueError(\n            f\"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome.\"\n        )\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,\n    )\n    logger.warning(\n        \"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\",\n        training_args.local_rank,\n        training_args.device,\n        training_args.n_gpu,\n        bool(training_args.local_rank != -1),\n        training_args.fp16,\n    )\n    logger.info(\"Training/evaluation parameters %s\", training_args)\n\n    # Set seed\n    set_seed(training_args.seed)\n\n    try:\n        num_labels = glue_tasks_num_labels[data_args.task_name]\n        output_mode = glue_output_modes[data_args.task_name]\n    except KeyError:\n        raise ValueError(\"Task not found: %s\" % (data_args.task_name))\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        num_labels=num_labels,\n        finetuning_task=data_args.task_name,\n        cache_dir=model_args.cache_dir,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n    )\n    model = AutoModelForSequenceClassification.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n    )\n\n    # Get datasets\n    train_dataset = (\n        GlueDataset(data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir) if training_args.do_train else None\n    )\n    eval_dataset = (\n        GlueDataset(data_args, tokenizer=tokenizer, mode=\"dev\", cache_dir=model_args.cache_dir)\n        if training_args.do_eval\n        else None\n    )\n    test_dataset = (\n        GlueDataset(data_args, tokenizer=tokenizer, mode=\"test\", cache_dir=model_args.cache_dir)\n        if training_args.do_predict\n        else None\n    )\n\n    def build_compute_metrics_fn(task_name: str) -> Callable[[EvalPrediction], Dict]:\n        def compute_metrics_fn(p: EvalPrediction):\n            preds = p.predictions[0] if type(p.predictions) == tuple else p.predictions\n            if output_mode == \"classification\":\n                preds = np.argmax(preds, axis=1)\n            else:  # regression\n                preds = np.squeeze(preds)\n            return glue_compute_metrics(task_name, preds, p.label_ids)\n\n        return compute_metrics_fn\n\n    # Initialize our Trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset,\n        eval_dataset=eval_dataset,\n        compute_metrics=build_compute_metrics_fn(data_args.task_name),\n    )\n\n    # Training\n    if training_args.do_train:\n        trainer.train(\n            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None\n        )\n        trainer.save_model()\n        # For convenience, we also re-save the tokenizer to the same directory,\n        # so that you can share your model easily on huggingface.co/models =)\n        if trainer.is_world_master():\n            tokenizer.save_pretrained(training_args.output_dir)\n\n    # Evaluation\n    eval_results = {}\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n\n        # Loop to handle MNLI double evaluation (matched, mis-matched)\n        eval_datasets = [eval_dataset]\n        if data_args.task_name == \"mnli\":\n            mnli_mm_data_args = dataclasses.replace(data_args, task_name=\"mnli-mm\")\n            eval_datasets.append(\n                GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, mode=\"dev\", cache_dir=model_args.cache_dir)\n            )\n\n        for eval_dataset in eval_datasets:\n            trainer.compute_metrics = build_compute_metrics_fn(eval_dataset.args.task_name)\n            eval_result = trainer.evaluate(eval_dataset=eval_dataset)\n\n            output_eval_file = os.path.join(\n                training_args.output_dir, f\"eval_results_{eval_dataset.args.task_name}.txt\"\n            )\n            if trainer.is_world_master():\n                with open(output_eval_file, \"w\") as writer:\n                    logger.info(\"***** Eval results {} *****\".format(eval_dataset.args.task_name))\n                    for key, value in eval_result.items():\n                        logger.info(\"  %s = %s\", key, value)\n                        writer.write(\"%s = %s\\n\" % (key, value))\n\n            eval_results.update(eval_result)\n\n    if training_args.do_predict:\n        logging.info(\"*** Test ***\")\n        test_datasets = [test_dataset]\n        if data_args.task_name == \"mnli\":\n            mnli_mm_data_args = dataclasses.replace(data_args, task_name=\"mnli-mm\")\n            test_datasets.append(\n                GlueDataset(mnli_mm_data_args, tokenizer=tokenizer, mode=\"test\", cache_dir=model_args.cache_dir)\n            )\n\n        for test_dataset in test_datasets:\n            predictions = trainer.predict(test_dataset=test_dataset).predictions\n            if output_mode == \"classification\":\n                predictions = np.argmax(predictions, axis=1)\n\n            output_test_file = os.path.join(\n                training_args.output_dir, f\"test_results_{test_dataset.args.task_name}.txt\"\n            )\n            if trainer.is_world_master():\n                with open(output_test_file, \"w\") as writer:\n                    logger.info(\"***** Test results {} *****\".format(test_dataset.args.task_name))\n                    writer.write(\"index\\tprediction\\n\")\n                    for index, item in enumerate(predictions):\n                        if output_mode == \"regression\":\n                            writer.write(\"%d\\t%3.3f\\n\" % (index, item))\n                        else:\n                            item = test_dataset.get_labels()[item]\n                            writer.write(\"%d\\t%s\\n\" % (index, item))\n    return eval_results\n\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n    main()\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "examples/text-classification/run_glue.py"}]}