We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Multi machine training not working
I am trying to run my training code on 2 machines. Each of them has 2 GPUs. However, it seems the program runs separately and do not fasten the training progress. Here is my `config.yaml`

machine 1:

```
compute_environment: LOCAL_MACHINE
distributed_type: MULTI_GPU
fp16: true
machine_rank: 0
main_process_ip: 192.168.0.1
main_process_port: 99999
main_training_function: main
num_machines: 2
num_processes: 4

```

machine 2:

```
compute_environment: LOCAL_MACHINE
distributed_type: MULTI_GPU
fp16: true
machine_rank: 1
main_process_ip: 192.168.0.1
main_process_port: 99999
main_training_function: main
num_machines: 2
num_processes: 4

```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/accelerate/commands/launch.py`
Content:
```
1 #!/usr/bin/env python
2 
3 # Copyright 2021 The HuggingFace Team. All rights reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 
17 import argparse
18 import importlib
19 import inspect
20 import os
21 import subprocess
22 import sys
23 from ast import literal_eval
24 from pathlib import Path
25 from typing import Dict, List
26 
27 from accelerate.commands.config import default_config_file, load_config_from_file
28 from accelerate.commands.config.config_args import SageMakerConfig
29 from accelerate.state import ComputeEnvironment, DistributedType
30 from accelerate.utils import is_sagemaker_available
31 
32 
33 class _AddOneArg:
34     def __init__(self, launcher):
35         self.launcher = launcher
36 
37     def __call__(self, index):
38         self.launcher()
39 
40 
41 def launch_command_parser(subparsers=None):
42     if subparsers is not None:
43         parser = subparsers.add_parser("launch")
44     else:
45         parser = argparse.ArgumentParser("Accelerate launch command")
46 
47     parser.add_argument(
48         "--config_file", default=None, help="The config file to use for the default values in the launching script."
49     )
50     parser.add_argument(
51         "--multi_gpu",
52         default=False,
53         action="store_true",
54         help="Whether or not this should launch a distributed GPU training.",
55     )
56     parser.add_argument(
57         "--tpu", default=False, action="store_true", help="Whether or not this should launch a TPU training."
58     )
59     parser.add_argument(
60         "--fp16", default=False, action="store_true", help="Whether or not to use mixed precision training."
61     )
62     parser.add_argument(
63         "--cpu", default=False, action="store_true", help="Whether or not to force the training on the CPU."
64     )
65     parser.add_argument(
66         "--num_processes", type=int, default=None, help="The total number of processes to be launched in parallel."
67     )
68     parser.add_argument(
69         "--num_machines", type=int, default=1, help="The total number of machines used in this training."
70     )
71     parser.add_argument(
72         "--machine_rank", type=int, default=0, help="The rank of the machine on which this script is launched."
73     )
74     parser.add_argument("--main_process_ip", type=str, default=None, help="The IP address of the machine of rank 0.")
75     parser.add_argument(
76         "--main_process_port",
77         type=int,
78         default=None,
79         help="The port to use to communicate with the machine of rank 0.",
80     )
81     parser.add_argument(
82         "--main_training_function",
83         type=str,
84         default=None,
85         help="The name of the main function to be executed in your script (only for TPU training).",
86     )
87     parser.add_argument(
88         "--aws_access_key_id",
89         type=str,
90         default=None,
91         help="The AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training job",
92     )
93     parser.add_argument(
94         "--aws_secret_access_key",
95         type=str,
96         default=None,
97         help="The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job",
98     )
99     parser.add_argument(
100         "training_script",
101         type=str,
102         help=(
103             "The full path to the script to be launched in parallel, followed by all the arguments for the training "
104             "script."
105         ),
106     )
107     # Other arguments of the training scripts
108     parser.add_argument("training_script_args", nargs=argparse.REMAINDER, help="Arguments of the training script.")
109 
110     if subparsers is not None:
111         parser.set_defaults(func=launch_command)
112     return parser
113 
114 
115 def simple_launcher(args):
116     cmd = [sys.executable, args.training_script]
117     cmd.extend(args.training_script_args)
118 
119     current_env = os.environ.copy()
120     current_env["USE_CPU"] = str(args.cpu)
121     current_env["USE_FP16"] = str(args.fp16)
122 
123     process = subprocess.Popen(cmd, env=current_env)
124     process.wait()
125     if process.returncode != 0:
126         raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
127 
128 
129 def multi_gpu_launcher(args):
130     cmd = [sys.executable, "-m", "torch.distributed.launch"]
131     cmd.extend(["--nproc_per_node", str(args.num_processes), "--use_env"])
132     if args.num_machines > 1:
133         cmd.extend(
134             [
135                 "--nproc_per_node",
136                 str(args.num_processes // args.num_machines),
137                 "--nnodes",
138                 str(args.num_machines),
139                 "--node_rank",
140                 str(args.machine_rank),
141                 "--master_addr",
142                 args.main_process_ip,
143                 "--node_rank",
144                 str(args.main_process_port),
145             ]
146         )
147     else:
148         cmd.extend(["--nproc_per_node", str(args.num_processes)])
149     cmd.append(args.training_script)
150     cmd.extend(args.training_script_args)
151 
152     current_env = os.environ.copy()
153     current_env["USE_FP16"] = str(args.fp16)
154 
155     process = subprocess.Popen(cmd, env=current_env)
156     process.wait()
157     if process.returncode != 0:
158         raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
159 
160 
161 def tpu_launcher(args):
162     import torch_xla.distributed.xla_multiprocessing as xmp
163 
164     # Import training_script as a module.
165     script_path = Path(args.training_script)
166     sys.path.append(str(script_path.parent.resolve()))
167     mod_name = script_path.stem
168     mod = importlib.import_module(mod_name)
169     if not hasattr(mod, args.main_training_function):
170         raise ValueError(
171             f"Your training script should have a function named {args.main_training_function}, or you should pass a "
172             "different value to `--main_training_function`."
173         )
174     main_function = getattr(mod, args.main_training_function)
175 
176     # Patch sys.argv
177     sys.argv = [args.training_script] + args.training_script_args
178 
179     # If the function does not take one argument, launch will fail
180     launcher_sig = inspect.signature(main_function)
181     if len(launcher_sig.parameters) == 0:
182         xmp.spawn(_AddOneArg(main_function), args=(), nprocs=args.num_processes)
183     else:
184         xmp.spawn(main_function, args=(), nprocs=args.num_processes)
185 
186 
187 def _convert_nargs_to_dict(nargs: List[str]) -> Dict[str, str]:
188     if len(nargs) < 0:
189         return {}
190     # helper function to infer type for argsparser
191 
192     def _infer_type(s):
193         try:
194             s = float(s)
195 
196             if s // 1 == s:
197                 return int(s)
198             return s
199         except ValueError:
200             return s
201 
202     parser = argparse.ArgumentParser()
203     _, unknown = parser.parse_known_args(nargs)
204     for index, argument in enumerate(unknown):
205         if argument.startswith(("-", "--")):
206             action = None
207             if index + 1 < len(unknown):  # checks if next index would be in list
208                 if unknown[index + 1].startswith(("-", "--")):  # checks if next element is an key
209                     # raise an error if element is store_true or store_false
210                     raise ValueError(
211                         "SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types"
212                     )
213             else:  # raise an error if last element is store_true or store_false
214                 raise ValueError(
215                     "SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types"
216                 )
217             # adds argument to parser based on action_store true
218             if action is None:
219                 parser.add_argument(argument, type=_infer_type)
220             else:
221                 parser.add_argument(argument, action=action)
222 
223     return {
224         key: (literal_eval(value) if value == "True" or value == "False" else value)
225         for key, value in parser.parse_args(nargs).__dict__.items()
226     }
227 
228 
229 def sagemaker_launcher(sagemaker_config: SageMakerConfig, args):
230     if not is_sagemaker_available():
231         raise ImportError(
232             "Please install sagemaker to be able to launch training on Amazon SageMaker with `pip install accelerate[sagemaker]`"
233         )
234     from sagemaker.huggingface import HuggingFace
235 
236     # configure environment
237     print("Configuring Amazon SageMaker environment")
238     os.environ["AWS_DEFAULT_REGION"] = sagemaker_config.region
239 
240     # configure credentials
241     if sagemaker_config.profile is not None:
242         os.environ["AWS_PROFILE"] = sagemaker_config.profile
243     elif args.aws_access_key_id is not None and args.aws_secret_access_key is not None:
244         os.environ["AWS_ACCESS_KEY_ID"] = args.aws_access_key_id
245         os.environ["AWS_SECRET_ACCESS_KEY"] = args.aws_secret_access_key
246     else:
247         raise EnvironmentError(
248             "You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profile"
249         )
250 
251     # extract needed arguments
252     source_dir = os.path.dirname(args.training_script)
253     if not source_dir:  # checks if string is empty
254         source_dir = "."
255     entry_point = os.path.basename(args.training_script)
256     if not entry_point.endswith(".py"):
257         raise ValueError(f'Your training script should be a python script and not "{entry_point}"')
258 
259     print("Converting Arguments to Hyperparameters")
260     hyperparameters = _convert_nargs_to_dict(args.training_script_args)
261 
262     environment = {"USE_FP16": args.fp16}  # Environment variables to be set for use during training job
263 
264     # configure distribution set up
265     distribution = None  # TODO: not yet implemented
266 
267     # configure session
268     print("Creating Estimator")
269     huggingface_estimator = HuggingFace(
270         entry_point=entry_point,
271         source_dir=source_dir,
272         role=sagemaker_config.iam_role_name,
273         transformers_version="4.4",
274         pytorch_version="1.6",
275         py_version="py36",
276         base_job_name=sagemaker_config.base_job_name,
277         instance_count=sagemaker_config.num_machines,
278         instance_type=sagemaker_config.ec2_instance_type,
279         debugger_hook_config=False,
280         distribution=distribution,
281         hyperparameters=hyperparameters,
282         environment=environment,
283     )
284 
285     huggingface_estimator.fit()
286     print(f"You can find your model data at: {huggingface_estimator.model_data}")
287 
288 
289 def launch_command(args):
290     # Sanity checks
291     if args.multi_gpu and args.tpu:
292         raise ValueError("You can only pick one between `--multi_gpu` and `--tpu`.")
293 
294     defaults = None
295     # Get the default from the config file.
296     if args.config_file is not None or os.path.isfile(default_config_file) and not args.cpu:
297         defaults = load_config_from_file(args.config_file)
298         if not args.multi_gpu and not args.tpu:
299             args.multi_gpu = defaults.distributed_type == DistributedType.MULTI_GPU
300             args.tpu = defaults.distributed_type == DistributedType.TPU
301         if args.num_processes is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
302             args.num_processes = defaults.num_processes
303         if not args.fp16:
304             args.fp16 = defaults.fp16
305         if args.main_training_function is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
306             args.main_training_function = defaults.main_training_function
307     else:
308         if args.num_processes is None:
309             args.num_processes = 1
310 
311     # Use the proper launcher
312     if args.multi_gpu and not args.cpu:
313         multi_gpu_launcher(args)
314     elif args.tpu and not args.cpu:
315         tpu_launcher(args)
316     elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
317         sagemaker_launcher(defaults, args)
318     else:
319         simple_launcher(args)
320 
321 
322 def main():
323     parser = launch_command_parser()
324     args = parser.parse_args()
325     launch_command(args)
326 
327 
328 if __name__ == "__main__":
329     main()
330
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -66,10 +66,10 @@
         "--num_processes", type=int, default=None, help="The total number of processes to be launched in parallel."
     )
     parser.add_argument(
-        "--num_machines", type=int, default=1, help="The total number of machines used in this training."
+        "--num_machines", type=int, default=None, help="The total number of machines used in this training."
     )
     parser.add_argument(
-        "--machine_rank", type=int, default=0, help="The rank of the machine on which this script is launched."
+        "--machine_rank", type=int, default=None, help="The rank of the machine on which this script is launched."
     )
     parser.add_argument("--main_process_ip", type=str, default=None, help="The IP address of the machine of rank 0.")
     parser.add_argument(
@@ -298,6 +298,16 @@
         if not args.multi_gpu and not args.tpu:
             args.multi_gpu = defaults.distributed_type == DistributedType.MULTI_GPU
             args.tpu = defaults.distributed_type == DistributedType.TPU
+        if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
+            # Update args with the defaults
+            for name, attr in defaults.__dict__.items():
+                # Those args are handled separately
+                if (
+                    name not in ["compute_environment", "fp16", "distributed_type"]
+                    and getattr(args, name, None) is None
+                ):
+                    setattr(args, name, attr)
+
         if args.num_processes is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
             args.num_processes = defaults.num_processes
         if not args.fp16:

{"golden_diff": "diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py\n--- a/src/accelerate/commands/launch.py\n+++ b/src/accelerate/commands/launch.py\n@@ -66,10 +66,10 @@\n         \"--num_processes\", type=int, default=None, help=\"The total number of processes to be launched in parallel.\"\n     )\n     parser.add_argument(\n-        \"--num_machines\", type=int, default=1, help=\"The total number of machines used in this training.\"\n+        \"--num_machines\", type=int, default=None, help=\"The total number of machines used in this training.\"\n     )\n     parser.add_argument(\n-        \"--machine_rank\", type=int, default=0, help=\"The rank of the machine on which this script is launched.\"\n+        \"--machine_rank\", type=int, default=None, help=\"The rank of the machine on which this script is launched.\"\n     )\n     parser.add_argument(\"--main_process_ip\", type=str, default=None, help=\"The IP address of the machine of rank 0.\")\n     parser.add_argument(\n@@ -298,6 +298,16 @@\n         if not args.multi_gpu and not args.tpu:\n             args.multi_gpu = defaults.distributed_type == DistributedType.MULTI_GPU\n             args.tpu = defaults.distributed_type == DistributedType.TPU\n+        if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n+            # Update args with the defaults\n+            for name, attr in defaults.__dict__.items():\n+                # Those args are handled separately\n+                if (\n+                    name not in [\"compute_environment\", \"fp16\", \"distributed_type\"]\n+                    and getattr(args, name, None) is None\n+                ):\n+                    setattr(args, name, attr)\n+\n         if args.num_processes is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n             args.num_processes = defaults.num_processes\n         if not args.fp16:\n", "issue": "Multi machine training not working\nI am trying to run my training code on 2 machines. Each of them has 2 GPUs. However, it seems the program runs separately and do not fasten the training progress. Here is my `config.yaml`\r\n\r\nmachine 1:\r\n\r\n```\r\ncompute_environment: LOCAL_MACHINE\r\ndistributed_type: MULTI_GPU\r\nfp16: true\r\nmachine_rank: 0\r\nmain_process_ip: 192.168.0.1\r\nmain_process_port: 99999\r\nmain_training_function: main\r\nnum_machines: 2\r\nnum_processes: 4\r\n\r\n```\r\n\r\nmachine 2:\r\n\r\n```\r\ncompute_environment: LOCAL_MACHINE\r\ndistributed_type: MULTI_GPU\r\nfp16: true\r\nmachine_rank: 1\r\nmain_process_ip: 192.168.0.1\r\nmain_process_port: 99999\r\nmain_training_function: main\r\nnum_machines: 2\r\nnum_processes: 4\r\n\r\n```\n", "before_files": [{"content": "#!/usr/bin/env python\n\n# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport argparse\nimport importlib\nimport inspect\nimport os\nimport subprocess\nimport sys\nfrom ast import literal_eval\nfrom pathlib import Path\nfrom typing import Dict, List\n\nfrom accelerate.commands.config import default_config_file, load_config_from_file\nfrom accelerate.commands.config.config_args import SageMakerConfig\nfrom accelerate.state import ComputeEnvironment, DistributedType\nfrom accelerate.utils import is_sagemaker_available\n\n\nclass _AddOneArg:\n    def __init__(self, launcher):\n        self.launcher = launcher\n\n    def __call__(self, index):\n        self.launcher()\n\n\ndef launch_command_parser(subparsers=None):\n    if subparsers is not None:\n        parser = subparsers.add_parser(\"launch\")\n    else:\n        parser = argparse.ArgumentParser(\"Accelerate launch command\")\n\n    parser.add_argument(\n        \"--config_file\", default=None, help=\"The config file to use for the default values in the launching script.\"\n    )\n    parser.add_argument(\n        \"--multi_gpu\",\n        default=False,\n        action=\"store_true\",\n        help=\"Whether or not this should launch a distributed GPU training.\",\n    )\n    parser.add_argument(\n        \"--tpu\", default=False, action=\"store_true\", help=\"Whether or not this should launch a TPU training.\"\n    )\n    parser.add_argument(\n        \"--fp16\", default=False, action=\"store_true\", help=\"Whether or not to use mixed precision training.\"\n    )\n    parser.add_argument(\n        \"--cpu\", default=False, action=\"store_true\", help=\"Whether or not to force the training on the CPU.\"\n    )\n    parser.add_argument(\n        \"--num_processes\", type=int, default=None, help=\"The total number of processes to be launched in parallel.\"\n    )\n    parser.add_argument(\n        \"--num_machines\", type=int, default=1, help=\"The total number of machines used in this training.\"\n    )\n    parser.add_argument(\n        \"--machine_rank\", type=int, default=0, help=\"The rank of the machine on which this script is launched.\"\n    )\n    parser.add_argument(\"--main_process_ip\", type=str, default=None, help=\"The IP address of the machine of rank 0.\")\n    parser.add_argument(\n        \"--main_process_port\",\n        type=int,\n        default=None,\n        help=\"The port to use to communicate with the machine of rank 0.\",\n    )\n    parser.add_argument(\n        \"--main_training_function\",\n        type=str,\n        default=None,\n        help=\"The name of the main function to be executed in your script (only for TPU training).\",\n    )\n    parser.add_argument(\n        \"--aws_access_key_id\",\n        type=str,\n        default=None,\n        help=\"The AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training job\",\n    )\n    parser.add_argument(\n        \"--aws_secret_access_key\",\n        type=str,\n        default=None,\n        help=\"The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job\",\n    )\n    parser.add_argument(\n        \"training_script\",\n        type=str,\n        help=(\n            \"The full path to the script to be launched in parallel, followed by all the arguments for the training \"\n            \"script.\"\n        ),\n    )\n    # Other arguments of the training scripts\n    parser.add_argument(\"training_script_args\", nargs=argparse.REMAINDER, help=\"Arguments of the training script.\")\n\n    if subparsers is not None:\n        parser.set_defaults(func=launch_command)\n    return parser\n\n\ndef simple_launcher(args):\n    cmd = [sys.executable, args.training_script]\n    cmd.extend(args.training_script_args)\n\n    current_env = os.environ.copy()\n    current_env[\"USE_CPU\"] = str(args.cpu)\n    current_env[\"USE_FP16\"] = str(args.fp16)\n\n    process = subprocess.Popen(cmd, env=current_env)\n    process.wait()\n    if process.returncode != 0:\n        raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n\n\ndef multi_gpu_launcher(args):\n    cmd = [sys.executable, \"-m\", \"torch.distributed.launch\"]\n    cmd.extend([\"--nproc_per_node\", str(args.num_processes), \"--use_env\"])\n    if args.num_machines > 1:\n        cmd.extend(\n            [\n                \"--nproc_per_node\",\n                str(args.num_processes // args.num_machines),\n                \"--nnodes\",\n                str(args.num_machines),\n                \"--node_rank\",\n                str(args.machine_rank),\n                \"--master_addr\",\n                args.main_process_ip,\n                \"--node_rank\",\n                str(args.main_process_port),\n            ]\n        )\n    else:\n        cmd.extend([\"--nproc_per_node\", str(args.num_processes)])\n    cmd.append(args.training_script)\n    cmd.extend(args.training_script_args)\n\n    current_env = os.environ.copy()\n    current_env[\"USE_FP16\"] = str(args.fp16)\n\n    process = subprocess.Popen(cmd, env=current_env)\n    process.wait()\n    if process.returncode != 0:\n        raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n\n\ndef tpu_launcher(args):\n    import torch_xla.distributed.xla_multiprocessing as xmp\n\n    # Import training_script as a module.\n    script_path = Path(args.training_script)\n    sys.path.append(str(script_path.parent.resolve()))\n    mod_name = script_path.stem\n    mod = importlib.import_module(mod_name)\n    if not hasattr(mod, args.main_training_function):\n        raise ValueError(\n            f\"Your training script should have a function named {args.main_training_function}, or you should pass a \"\n            \"different value to `--main_training_function`.\"\n        )\n    main_function = getattr(mod, args.main_training_function)\n\n    # Patch sys.argv\n    sys.argv = [args.training_script] + args.training_script_args\n\n    # If the function does not take one argument, launch will fail\n    launcher_sig = inspect.signature(main_function)\n    if len(launcher_sig.parameters) == 0:\n        xmp.spawn(_AddOneArg(main_function), args=(), nprocs=args.num_processes)\n    else:\n        xmp.spawn(main_function, args=(), nprocs=args.num_processes)\n\n\ndef _convert_nargs_to_dict(nargs: List[str]) -> Dict[str, str]:\n    if len(nargs) < 0:\n        return {}\n    # helper function to infer type for argsparser\n\n    def _infer_type(s):\n        try:\n            s = float(s)\n\n            if s // 1 == s:\n                return int(s)\n            return s\n        except ValueError:\n            return s\n\n    parser = argparse.ArgumentParser()\n    _, unknown = parser.parse_known_args(nargs)\n    for index, argument in enumerate(unknown):\n        if argument.startswith((\"-\", \"--\")):\n            action = None\n            if index + 1 < len(unknown):  # checks if next index would be in list\n                if unknown[index + 1].startswith((\"-\", \"--\")):  # checks if next element is an key\n                    # raise an error if element is store_true or store_false\n                    raise ValueError(\n                        \"SageMaker doesn\u2019t support argparse actions for `store_true` or `store_false`. Please define explicit types\"\n                    )\n            else:  # raise an error if last element is store_true or store_false\n                raise ValueError(\n                    \"SageMaker doesn\u2019t support argparse actions for `store_true` or `store_false`. Please define explicit types\"\n                )\n            # adds argument to parser based on action_store true\n            if action is None:\n                parser.add_argument(argument, type=_infer_type)\n            else:\n                parser.add_argument(argument, action=action)\n\n    return {\n        key: (literal_eval(value) if value == \"True\" or value == \"False\" else value)\n        for key, value in parser.parse_args(nargs).__dict__.items()\n    }\n\n\ndef sagemaker_launcher(sagemaker_config: SageMakerConfig, args):\n    if not is_sagemaker_available():\n        raise ImportError(\n            \"Please install sagemaker to be able to launch training on Amazon SageMaker with `pip install accelerate[sagemaker]`\"\n        )\n    from sagemaker.huggingface import HuggingFace\n\n    # configure environment\n    print(\"Configuring Amazon SageMaker environment\")\n    os.environ[\"AWS_DEFAULT_REGION\"] = sagemaker_config.region\n\n    # configure credentials\n    if sagemaker_config.profile is not None:\n        os.environ[\"AWS_PROFILE\"] = sagemaker_config.profile\n    elif args.aws_access_key_id is not None and args.aws_secret_access_key is not None:\n        os.environ[\"AWS_ACCESS_KEY_ID\"] = args.aws_access_key_id\n        os.environ[\"AWS_SECRET_ACCESS_KEY\"] = args.aws_secret_access_key\n    else:\n        raise EnvironmentError(\n            \"You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profile\"\n        )\n\n    # extract needed arguments\n    source_dir = os.path.dirname(args.training_script)\n    if not source_dir:  # checks if string is empty\n        source_dir = \".\"\n    entry_point = os.path.basename(args.training_script)\n    if not entry_point.endswith(\".py\"):\n        raise ValueError(f'Your training script should be a python script and not \"{entry_point}\"')\n\n    print(\"Converting Arguments to Hyperparameters\")\n    hyperparameters = _convert_nargs_to_dict(args.training_script_args)\n\n    environment = {\"USE_FP16\": args.fp16}  # Environment variables to be set for use during training job\n\n    # configure distribution set up\n    distribution = None  # TODO: not yet implemented\n\n    # configure session\n    print(\"Creating Estimator\")\n    huggingface_estimator = HuggingFace(\n        entry_point=entry_point,\n        source_dir=source_dir,\n        role=sagemaker_config.iam_role_name,\n        transformers_version=\"4.4\",\n        pytorch_version=\"1.6\",\n        py_version=\"py36\",\n        base_job_name=sagemaker_config.base_job_name,\n        instance_count=sagemaker_config.num_machines,\n        instance_type=sagemaker_config.ec2_instance_type,\n        debugger_hook_config=False,\n        distribution=distribution,\n        hyperparameters=hyperparameters,\n        environment=environment,\n    )\n\n    huggingface_estimator.fit()\n    print(f\"You can find your model data at: {huggingface_estimator.model_data}\")\n\n\ndef launch_command(args):\n    # Sanity checks\n    if args.multi_gpu and args.tpu:\n        raise ValueError(\"You can only pick one between `--multi_gpu` and `--tpu`.\")\n\n    defaults = None\n    # Get the default from the config file.\n    if args.config_file is not None or os.path.isfile(default_config_file) and not args.cpu:\n        defaults = load_config_from_file(args.config_file)\n        if not args.multi_gpu and not args.tpu:\n            args.multi_gpu = defaults.distributed_type == DistributedType.MULTI_GPU\n            args.tpu = defaults.distributed_type == DistributedType.TPU\n        if args.num_processes is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n            args.num_processes = defaults.num_processes\n        if not args.fp16:\n            args.fp16 = defaults.fp16\n        if args.main_training_function is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n            args.main_training_function = defaults.main_training_function\n    else:\n        if args.num_processes is None:\n            args.num_processes = 1\n\n    # Use the proper launcher\n    if args.multi_gpu and not args.cpu:\n        multi_gpu_launcher(args)\n    elif args.tpu and not args.cpu:\n        tpu_launcher(args)\n    elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:\n        sagemaker_launcher(defaults, args)\n    else:\n        simple_launcher(args)\n\n\ndef main():\n    parser = launch_command_parser()\n    args = parser.parse_args()\n    launch_command(args)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "src/accelerate/commands/launch.py"}], "after_files": [{"content": "#!/usr/bin/env python\n\n# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport argparse\nimport importlib\nimport inspect\nimport os\nimport subprocess\nimport sys\nfrom ast import literal_eval\nfrom pathlib import Path\nfrom typing import Dict, List\n\nfrom accelerate.commands.config import default_config_file, load_config_from_file\nfrom accelerate.commands.config.config_args import SageMakerConfig\nfrom accelerate.state import ComputeEnvironment, DistributedType\nfrom accelerate.utils import is_sagemaker_available\n\n\nclass _AddOneArg:\n    def __init__(self, launcher):\n        self.launcher = launcher\n\n    def __call__(self, index):\n        self.launcher()\n\n\ndef launch_command_parser(subparsers=None):\n    if subparsers is not None:\n        parser = subparsers.add_parser(\"launch\")\n    else:\n        parser = argparse.ArgumentParser(\"Accelerate launch command\")\n\n    parser.add_argument(\n        \"--config_file\", default=None, help=\"The config file to use for the default values in the launching script.\"\n    )\n    parser.add_argument(\n        \"--multi_gpu\",\n        default=False,\n        action=\"store_true\",\n        help=\"Whether or not this should launch a distributed GPU training.\",\n    )\n    parser.add_argument(\n        \"--tpu\", default=False, action=\"store_true\", help=\"Whether or not this should launch a TPU training.\"\n    )\n    parser.add_argument(\n        \"--fp16\", default=False, action=\"store_true\", help=\"Whether or not to use mixed precision training.\"\n    )\n    parser.add_argument(\n        \"--cpu\", default=False, action=\"store_true\", help=\"Whether or not to force the training on the CPU.\"\n    )\n    parser.add_argument(\n        \"--num_processes\", type=int, default=None, help=\"The total number of processes to be launched in parallel.\"\n    )\n    parser.add_argument(\n        \"--num_machines\", type=int, default=None, help=\"The total number of machines used in this training.\"\n    )\n    parser.add_argument(\n        \"--machine_rank\", type=int, default=None, help=\"The rank of the machine on which this script is launched.\"\n    )\n    parser.add_argument(\"--main_process_ip\", type=str, default=None, help=\"The IP address of the machine of rank 0.\")\n    parser.add_argument(\n        \"--main_process_port\",\n        type=int,\n        default=None,\n        help=\"The port to use to communicate with the machine of rank 0.\",\n    )\n    parser.add_argument(\n        \"--main_training_function\",\n        type=str,\n        default=None,\n        help=\"The name of the main function to be executed in your script (only for TPU training).\",\n    )\n    parser.add_argument(\n        \"--aws_access_key_id\",\n        type=str,\n        default=None,\n        help=\"The AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training job\",\n    )\n    parser.add_argument(\n        \"--aws_secret_access_key\",\n        type=str,\n        default=None,\n        help=\"The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job\",\n    )\n    parser.add_argument(\n        \"training_script\",\n        type=str,\n        help=(\n            \"The full path to the script to be launched in parallel, followed by all the arguments for the training \"\n            \"script.\"\n        ),\n    )\n    # Other arguments of the training scripts\n    parser.add_argument(\"training_script_args\", nargs=argparse.REMAINDER, help=\"Arguments of the training script.\")\n\n    if subparsers is not None:\n        parser.set_defaults(func=launch_command)\n    return parser\n\n\ndef simple_launcher(args):\n    cmd = [sys.executable, args.training_script]\n    cmd.extend(args.training_script_args)\n\n    current_env = os.environ.copy()\n    current_env[\"USE_CPU\"] = str(args.cpu)\n    current_env[\"USE_FP16\"] = str(args.fp16)\n\n    process = subprocess.Popen(cmd, env=current_env)\n    process.wait()\n    if process.returncode != 0:\n        raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n\n\ndef multi_gpu_launcher(args):\n    cmd = [sys.executable, \"-m\", \"torch.distributed.launch\"]\n    cmd.extend([\"--nproc_per_node\", str(args.num_processes), \"--use_env\"])\n    if args.num_machines > 1:\n        cmd.extend(\n            [\n                \"--nproc_per_node\",\n                str(args.num_processes // args.num_machines),\n                \"--nnodes\",\n                str(args.num_machines),\n                \"--node_rank\",\n                str(args.machine_rank),\n                \"--master_addr\",\n                args.main_process_ip,\n                \"--node_rank\",\n                str(args.main_process_port),\n            ]\n        )\n    else:\n        cmd.extend([\"--nproc_per_node\", str(args.num_processes)])\n    cmd.append(args.training_script)\n    cmd.extend(args.training_script_args)\n\n    current_env = os.environ.copy()\n    current_env[\"USE_FP16\"] = str(args.fp16)\n\n    process = subprocess.Popen(cmd, env=current_env)\n    process.wait()\n    if process.returncode != 0:\n        raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n\n\ndef tpu_launcher(args):\n    import torch_xla.distributed.xla_multiprocessing as xmp\n\n    # Import training_script as a module.\n    script_path = Path(args.training_script)\n    sys.path.append(str(script_path.parent.resolve()))\n    mod_name = script_path.stem\n    mod = importlib.import_module(mod_name)\n    if not hasattr(mod, args.main_training_function):\n        raise ValueError(\n            f\"Your training script should have a function named {args.main_training_function}, or you should pass a \"\n            \"different value to `--main_training_function`.\"\n        )\n    main_function = getattr(mod, args.main_training_function)\n\n    # Patch sys.argv\n    sys.argv = [args.training_script] + args.training_script_args\n\n    # If the function does not take one argument, launch will fail\n    launcher_sig = inspect.signature(main_function)\n    if len(launcher_sig.parameters) == 0:\n        xmp.spawn(_AddOneArg(main_function), args=(), nprocs=args.num_processes)\n    else:\n        xmp.spawn(main_function, args=(), nprocs=args.num_processes)\n\n\ndef _convert_nargs_to_dict(nargs: List[str]) -> Dict[str, str]:\n    if len(nargs) < 0:\n        return {}\n    # helper function to infer type for argsparser\n\n    def _infer_type(s):\n        try:\n            s = float(s)\n\n            if s // 1 == s:\n                return int(s)\n            return s\n        except ValueError:\n            return s\n\n    parser = argparse.ArgumentParser()\n    _, unknown = parser.parse_known_args(nargs)\n    for index, argument in enumerate(unknown):\n        if argument.startswith((\"-\", \"--\")):\n            action = None\n            if index + 1 < len(unknown):  # checks if next index would be in list\n                if unknown[index + 1].startswith((\"-\", \"--\")):  # checks if next element is an key\n                    # raise an error if element is store_true or store_false\n                    raise ValueError(\n                        \"SageMaker doesn\u2019t support argparse actions for `store_true` or `store_false`. Please define explicit types\"\n                    )\n            else:  # raise an error if last element is store_true or store_false\n                raise ValueError(\n                    \"SageMaker doesn\u2019t support argparse actions for `store_true` or `store_false`. Please define explicit types\"\n                )\n            # adds argument to parser based on action_store true\n            if action is None:\n                parser.add_argument(argument, type=_infer_type)\n            else:\n                parser.add_argument(argument, action=action)\n\n    return {\n        key: (literal_eval(value) if value == \"True\" or value == \"False\" else value)\n        for key, value in parser.parse_args(nargs).__dict__.items()\n    }\n\n\ndef sagemaker_launcher(sagemaker_config: SageMakerConfig, args):\n    if not is_sagemaker_available():\n        raise ImportError(\n            \"Please install sagemaker to be able to launch training on Amazon SageMaker with `pip install accelerate[sagemaker]`\"\n        )\n    from sagemaker.huggingface import HuggingFace\n\n    # configure environment\n    print(\"Configuring Amazon SageMaker environment\")\n    os.environ[\"AWS_DEFAULT_REGION\"] = sagemaker_config.region\n\n    # configure credentials\n    if sagemaker_config.profile is not None:\n        os.environ[\"AWS_PROFILE\"] = sagemaker_config.profile\n    elif args.aws_access_key_id is not None and args.aws_secret_access_key is not None:\n        os.environ[\"AWS_ACCESS_KEY_ID\"] = args.aws_access_key_id\n        os.environ[\"AWS_SECRET_ACCESS_KEY\"] = args.aws_secret_access_key\n    else:\n        raise EnvironmentError(\n            \"You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profile\"\n        )\n\n    # extract needed arguments\n    source_dir = os.path.dirname(args.training_script)\n    if not source_dir:  # checks if string is empty\n        source_dir = \".\"\n    entry_point = os.path.basename(args.training_script)\n    if not entry_point.endswith(\".py\"):\n        raise ValueError(f'Your training script should be a python script and not \"{entry_point}\"')\n\n    print(\"Converting Arguments to Hyperparameters\")\n    hyperparameters = _convert_nargs_to_dict(args.training_script_args)\n\n    environment = {\"USE_FP16\": args.fp16}  # Environment variables to be set for use during training job\n\n    # configure distribution set up\n    distribution = None  # TODO: not yet implemented\n\n    # configure session\n    print(\"Creating Estimator\")\n    huggingface_estimator = HuggingFace(\n        entry_point=entry_point,\n        source_dir=source_dir,\n        role=sagemaker_config.iam_role_name,\n        transformers_version=\"4.4\",\n        pytorch_version=\"1.6\",\n        py_version=\"py36\",\n        base_job_name=sagemaker_config.base_job_name,\n        instance_count=sagemaker_config.num_machines,\n        instance_type=sagemaker_config.ec2_instance_type,\n        debugger_hook_config=False,\n        distribution=distribution,\n        hyperparameters=hyperparameters,\n        environment=environment,\n    )\n\n    huggingface_estimator.fit()\n    print(f\"You can find your model data at: {huggingface_estimator.model_data}\")\n\n\ndef launch_command(args):\n    # Sanity checks\n    if args.multi_gpu and args.tpu:\n        raise ValueError(\"You can only pick one between `--multi_gpu` and `--tpu`.\")\n\n    defaults = None\n    # Get the default from the config file.\n    if args.config_file is not None or os.path.isfile(default_config_file) and not args.cpu:\n        defaults = load_config_from_file(args.config_file)\n        if not args.multi_gpu and not args.tpu:\n            args.multi_gpu = defaults.distributed_type == DistributedType.MULTI_GPU\n            args.tpu = defaults.distributed_type == DistributedType.TPU\n        if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n            # Update args with the defaults\n            for name, attr in defaults.__dict__.items():\n                # Those args are handled separately\n                if (\n                    name not in [\"compute_environment\", \"fp16\", \"distributed_type\"]\n                    and getattr(args, name, None) is None\n                ):\n                    setattr(args, name, attr)\n\n        if args.num_processes is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n            args.num_processes = defaults.num_processes\n        if not args.fp16:\n            args.fp16 = defaults.fp16\n        if args.main_training_function is None and defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:\n            args.main_training_function = defaults.main_training_function\n    else:\n        if args.num_processes is None:\n            args.num_processes = 1\n\n    # Use the proper launcher\n    if args.multi_gpu and not args.cpu:\n        multi_gpu_launcher(args)\n    elif args.tpu and not args.cpu:\n        tpu_launcher(args)\n    elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:\n        sagemaker_launcher(defaults, args)\n    else:\n        simple_launcher(args)\n\n\ndef main():\n    parser = launch_command_parser()\n    args = parser.parse_args()\n    launch_command(args)\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "src/accelerate/commands/launch.py"}]}