We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Support multirun partial failure in Nevergrad sweeper
Context here: https://github.com/facebookresearch/hydra/issues/1377


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py`
Content:
```
1 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
2 from dataclasses import dataclass, field
3 from typing import Any, Dict, Optional
4 
5 from hydra.core.config_store import ConfigStore
6 
7 
8 @dataclass
9 class ScalarConfigSpec:
10     """Representation of all the options to define
11     a scalar.
12     """
13 
14     # lower bound if any
15     lower: Optional[float] = None
16 
17     # upper bound if any
18     upper: Optional[float] = None
19 
20     # initial value
21     # default to the middle point if completely bounded
22     init: Optional[float] = None
23 
24     # step size for an update
25     # defaults to 1 if unbounded
26     # or 1/6 of the range if completely bounded
27     step: Optional[float] = None
28 
29     # cast to integer
30     integer: bool = False
31 
32     # logarithmically distributed
33     log: bool = False
34 
35 
36 @dataclass
37 class OptimConf:
38 
39     # name of the Nevergrad optimizer to use. Here is a sample:
40     #   - "OnePlusOne" extremely simple and robust, especially at low budget, but
41     #     tends to converge early.
42     #   - "CMA" very good algorithm, but may require a significant budget (> 120)
43     #   - "TwoPointsDE": an algorithm good in a wide range of settings, for significant
44     #     budgets (> 120).
45     #   - "NGOpt" an algorithm aiming at identifying the best optimizer given your input
46     #     definition (updated regularly)
47     # find out more within nevergrad's documentation:
48     # https://github.com/facebookresearch/nevergrad/
49     optimizer: str = "NGOpt"
50 
51     # total number of function evaluations to perform
52     budget: int = 80
53 
54     # number of parallel workers for performing function evaluations
55     num_workers: int = 10
56 
57     # set to true if the function evaluations are noisy
58     noisy: bool = False
59 
60     # set to true for performing maximization instead of minimization
61     maximize: bool = False
62 
63     # optimization seed, for reproducibility
64     seed: Optional[int] = None
65 
66 
67 @dataclass
68 class NevergradSweeperConf:
69     _target_: str = (
70         "hydra_plugins.hydra_nevergrad_sweeper.nevergrad_sweeper.NevergradSweeper"
71     )
72 
73     # configuration of the optimizer
74     optim: OptimConf = OptimConf()
75 
76     # default parametrization of the search space
77     # can be specified:
78     # - as a string, like commandline arguments
79     # - as a list, for categorical variables
80     # - as a full scalar specification
81     parametrization: Dict[str, Any] = field(default_factory=dict)
82 
83 
84 ConfigStore.instance().store(
85     group="hydra/sweeper",
86     name="nevergrad",
87     node=NevergradSweeperConf,
88     provider="nevergrad",
89 )
90
```

Path: `plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py`
Content:
```
1 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
2 import logging
3 from typing import (
4     Any,
5     Dict,
6     List,
7     MutableMapping,
8     MutableSequence,
9     Optional,
10     Tuple,
11     Union,
12 )
13 
14 import nevergrad as ng
15 from hydra.core.override_parser.overrides_parser import OverridesParser
16 from hydra.core.override_parser.types import (
17     ChoiceSweep,
18     IntervalSweep,
19     Override,
20     Transformer,
21 )
22 from hydra.core.plugins import Plugins
23 from hydra.plugins.launcher import Launcher
24 from hydra.plugins.sweeper import Sweeper
25 from hydra.types import HydraContext, TaskFunction
26 from omegaconf import DictConfig, ListConfig, OmegaConf
27 
28 from .config import OptimConf, ScalarConfigSpec
29 
30 log = logging.getLogger(__name__)
31 
32 
33 def create_nevergrad_param_from_config(
34     config: Union[MutableSequence[Any], MutableMapping[str, Any]]
35 ) -> Any:
36     if isinstance(config, MutableSequence):
37         if isinstance(config, ListConfig):
38             config = OmegaConf.to_container(config, resolve=True)  # type: ignore
39         return ng.p.Choice(config)
40     if isinstance(config, MutableMapping):
41         specs = ScalarConfigSpec(**config)
42         init = ["init", "lower", "upper"]
43         init_params = {x: getattr(specs, x) for x in init}
44         if not specs.log:
45             scalar = ng.p.Scalar(**init_params)
46             if specs.step is not None:
47                 scalar.set_mutation(sigma=specs.step)
48         else:
49             if specs.step is not None:
50                 init_params["exponent"] = specs.step
51             scalar = ng.p.Log(**init_params)
52         if specs.integer:
53             scalar.set_integer_casting()
54         return scalar
55     return config
56 
57 
58 def create_nevergrad_parameter_from_override(override: Override) -> Any:
59     val = override.value()
60     if not override.is_sweep_override():
61         return val
62     if override.is_choice_sweep():
63         assert isinstance(val, ChoiceSweep)
64         vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]
65         if "ordered" in val.tags:
66             return ng.p.TransitionChoice(vals)
67         else:
68             return ng.p.Choice(vals)
69     elif override.is_range_sweep():
70         vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]
71         return ng.p.Choice(vals)
72     elif override.is_interval_sweep():
73         assert isinstance(val, IntervalSweep)
74         if "log" in val.tags:
75             scalar = ng.p.Log(lower=val.start, upper=val.end)
76         else:
77             scalar = ng.p.Scalar(lower=val.start, upper=val.end)  # type: ignore
78         if isinstance(val.start, int):
79             scalar.set_integer_casting()
80         return scalar
81 
82 
83 class NevergradSweeperImpl(Sweeper):
84     def __init__(
85         self,
86         optim: OptimConf,
87         parametrization: Optional[DictConfig],
88     ):
89         self.opt_config = optim
90         self.config: Optional[DictConfig] = None
91         self.launcher: Optional[Launcher] = None
92         self.hydra_context: Optional[HydraContext] = None
93         self.job_results = None
94         self.parametrization: Dict[str, Any] = {}
95         if parametrization is not None:
96             assert isinstance(parametrization, DictConfig)
97             self.parametrization = {
98                 str(x): create_nevergrad_param_from_config(y)
99                 for x, y in parametrization.items()
100             }
101         self.job_idx: Optional[int] = None
102 
103     def setup(
104         self,
105         *,
106         hydra_context: HydraContext,
107         task_function: TaskFunction,
108         config: DictConfig,
109     ) -> None:
110         self.job_idx = 0
111         self.config = config
112         self.hydra_context = hydra_context
113         self.launcher = Plugins.instance().instantiate_launcher(
114             hydra_context=hydra_context, task_function=task_function, config=config
115         )
116 
117     def sweep(self, arguments: List[str]) -> None:
118 
119         assert self.config is not None
120         assert self.launcher is not None
121         assert self.job_idx is not None
122         direction = -1 if self.opt_config.maximize else 1
123         name = "maximization" if self.opt_config.maximize else "minimization"
124         # Override the parametrization from commandline
125         params = dict(self.parametrization)
126 
127         parser = OverridesParser.create()
128         parsed = parser.parse_overrides(arguments)
129 
130         for override in parsed:
131             params[
132                 override.get_key_element()
133             ] = create_nevergrad_parameter_from_override(override)
134 
135         parametrization = ng.p.Dict(**params)
136         parametrization.function.deterministic = not self.opt_config.noisy
137         parametrization.random_state.seed(self.opt_config.seed)
138         # log and build the optimizer
139         opt = self.opt_config.optimizer
140         remaining_budget = self.opt_config.budget
141         nw = self.opt_config.num_workers
142         log.info(
143             f"NevergradSweeper(optimizer={opt}, budget={remaining_budget}, "
144             f"num_workers={nw}) {name}"
145         )
146         log.info(f"with parametrization {parametrization}")
147         log.info(f"Sweep output dir: {self.config.hydra.sweep.dir}")
148         optimizer = ng.optimizers.registry[opt](parametrization, remaining_budget, nw)
149         # loop!
150         all_returns: List[Any] = []
151         best: Tuple[float, ng.p.Parameter] = (float("inf"), parametrization)
152         while remaining_budget > 0:
153             batch = min(nw, remaining_budget)
154             remaining_budget -= batch
155             candidates = [optimizer.ask() for _ in range(batch)]
156             overrides = list(
157                 tuple(f"{x}={y}" for x, y in c.value.items()) for c in candidates
158             )
159             self.validate_batch_is_legal(overrides)
160             returns = self.launcher.launch(overrides, initial_job_idx=self.job_idx)
161             self.job_idx += len(returns)
162             # would have been nice to avoid waiting for all jobs to finish
163             # aka batch size Vs steady state (launching a new job whenever one is done)
164             for cand, ret in zip(candidates, returns):
165                 loss = direction * ret.return_value
166                 optimizer.tell(cand, loss)
167                 if loss < best[0]:
168                     best = (loss, cand)
169             all_returns.extend(returns)
170         recom = optimizer.provide_recommendation()
171         results_to_serialize = {
172             "name": "nevergrad",
173             "best_evaluated_params": best[1].value,
174             "best_evaluated_result": direction * best[0],
175         }
176         OmegaConf.save(
177             OmegaConf.create(results_to_serialize),
178             f"{self.config.hydra.sweep.dir}/optimization_results.yaml",
179         )
180         log.info(
181             "Best parameters: %s", " ".join(f"{x}={y}" for x, y in recom.value.items())
182         )
183
```

Path: `plugins/hydra_nevergrad_sweeper/example/my_app.py`
Content:
```
1 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
2 import logging
3 
4 import hydra
5 from omegaconf import DictConfig
6 
7 log = logging.getLogger(__name__)
8 
9 
10 @hydra.main(config_path=".", config_name="config")
11 def dummy_training(cfg: DictConfig) -> float:
12     """A dummy function to minimize
13     Minimum is 0.0 at:
14     lr = 0.12, dropout=0.33, db=mnist, batch_size=4
15     """
16     do = cfg.dropout
17     bs = cfg.batch_size
18     out = float(
19         abs(do - 0.33) + int(cfg.db == "mnist") + abs(cfg.lr - 0.12) + abs(bs - 4)
20     )
21     log.info(
22         f"dummy_training(dropout={do:.3f}, lr={cfg.lr:.3f}, db={cfg.db}, batch_size={bs}) = {out:.3f}",
23     )
24     return out
25 
26 
27 if __name__ == "__main__":
28     dummy_training()
29
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/plugins/hydra_nevergrad_sweeper/example/my_app.py b/plugins/hydra_nevergrad_sweeper/example/my_app.py
--- a/plugins/hydra_nevergrad_sweeper/example/my_app.py
+++ b/plugins/hydra_nevergrad_sweeper/example/my_app.py
@@ -21,6 +21,8 @@
     log.info(
         f"dummy_training(dropout={do:.3f}, lr={cfg.lr:.3f}, db={cfg.db}, batch_size={bs}) = {out:.3f}",
     )
+    if cfg.error:
+        raise RuntimeError("cfg.error is True")
     return out
 
 
diff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py
--- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py
+++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py
@@ -1,5 +1,6 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import logging
+import math
 from typing import (
     Any,
     Dict,
@@ -12,6 +13,7 @@
 )
 
 import nevergrad as ng
+from hydra.core import utils
 from hydra.core.override_parser.overrides_parser import OverridesParser
 from hydra.core.override_parser.types import (
     ChoiceSweep,
@@ -158,14 +160,32 @@
             )
             self.validate_batch_is_legal(overrides)
             returns = self.launcher.launch(overrides, initial_job_idx=self.job_idx)
-            self.job_idx += len(returns)
             # would have been nice to avoid waiting for all jobs to finish
             # aka batch size Vs steady state (launching a new job whenever one is done)
+            self.job_idx += len(returns)
+            # check job status and prepare losses
+            failures = 0
             for cand, ret in zip(candidates, returns):
-                loss = direction * ret.return_value
-                optimizer.tell(cand, loss)
-                if loss < best[0]:
-                    best = (loss, cand)
+                if ret.status == utils.JobStatus.COMPLETED:
+                    rectified_loss = direction * ret.return_value
+                else:
+                    rectified_loss = math.inf
+                    failures += 1
+                    try:
+                        ret.return_value
+                    except Exception as e:
+                        log.warning(f"Returning infinity for failed experiment: {e}")
+                optimizer.tell(cand, rectified_loss)
+                if rectified_loss < best[0]:
+                    best = (rectified_loss, cand)
+            # raise if too many failures
+            if failures / len(returns) > self.opt_config.max_failure_rate:
+                log.error(
+                    f"Failed {failures} times out of {len(returns)} "
+                    f"with max_failure_rate={self.opt_config.max_failure_rate}"
+                )
+                for ret in returns:
+                    ret.return_value  # delegate raising to JobReturn, with actual traceback
             all_returns.extend(returns)
         recom = optimizer.provide_recommendation()
         results_to_serialize = {
diff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py
--- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py
+++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py
@@ -63,6 +63,9 @@
     # optimization seed, for reproducibility
     seed: Optional[int] = None
 
+    # maximum authorized failure rate for a batch of parameters
+    max_failure_rate: float = 0.0
+
 
 @dataclass
 class NevergradSweeperConf:

{"golden_diff": "diff --git a/plugins/hydra_nevergrad_sweeper/example/my_app.py b/plugins/hydra_nevergrad_sweeper/example/my_app.py\n--- a/plugins/hydra_nevergrad_sweeper/example/my_app.py\n+++ b/plugins/hydra_nevergrad_sweeper/example/my_app.py\n@@ -21,6 +21,8 @@\n     log.info(\n         f\"dummy_training(dropout={do:.3f}, lr={cfg.lr:.3f}, db={cfg.db}, batch_size={bs}) = {out:.3f}\",\n     )\n+    if cfg.error:\n+        raise RuntimeError(\"cfg.error is True\")\n     return out\n \n \ndiff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py\n--- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py\n+++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py\n@@ -1,5 +1,6 @@\n # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n import logging\n+import math\n from typing import (\n     Any,\n     Dict,\n@@ -12,6 +13,7 @@\n )\n \n import nevergrad as ng\n+from hydra.core import utils\n from hydra.core.override_parser.overrides_parser import OverridesParser\n from hydra.core.override_parser.types import (\n     ChoiceSweep,\n@@ -158,14 +160,32 @@\n             )\n             self.validate_batch_is_legal(overrides)\n             returns = self.launcher.launch(overrides, initial_job_idx=self.job_idx)\n-            self.job_idx += len(returns)\n             # would have been nice to avoid waiting for all jobs to finish\n             # aka batch size Vs steady state (launching a new job whenever one is done)\n+            self.job_idx += len(returns)\n+            # check job status and prepare losses\n+            failures = 0\n             for cand, ret in zip(candidates, returns):\n-                loss = direction * ret.return_value\n-                optimizer.tell(cand, loss)\n-                if loss < best[0]:\n-                    best = (loss, cand)\n+                if ret.status == utils.JobStatus.COMPLETED:\n+                    rectified_loss = direction * ret.return_value\n+                else:\n+                    rectified_loss = math.inf\n+                    failures += 1\n+                    try:\n+                        ret.return_value\n+                    except Exception as e:\n+                        log.warning(f\"Returning infinity for failed experiment: {e}\")\n+                optimizer.tell(cand, rectified_loss)\n+                if rectified_loss < best[0]:\n+                    best = (rectified_loss, cand)\n+            # raise if too many failures\n+            if failures / len(returns) > self.opt_config.max_failure_rate:\n+                log.error(\n+                    f\"Failed {failures} times out of {len(returns)} \"\n+                    f\"with max_failure_rate={self.opt_config.max_failure_rate}\"\n+                )\n+                for ret in returns:\n+                    ret.return_value  # delegate raising to JobReturn, with actual traceback\n             all_returns.extend(returns)\n         recom = optimizer.provide_recommendation()\n         results_to_serialize = {\ndiff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py\n--- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py\n+++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py\n@@ -63,6 +63,9 @@\n     # optimization seed, for reproducibility\n     seed: Optional[int] = None\n \n+    # maximum authorized failure rate for a batch of parameters\n+    max_failure_rate: float = 0.0\n+\n \n @dataclass\n class NevergradSweeperConf:\n", "issue": "Support multirun partial failure in Nevergrad sweeper\nContext here: https://github.com/facebookresearch/hydra/issues/1377\r\n\n", "before_files": [{"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, Optional\n\nfrom hydra.core.config_store import ConfigStore\n\n\n@dataclass\nclass ScalarConfigSpec:\n    \"\"\"Representation of all the options to define\n    a scalar.\n    \"\"\"\n\n    # lower bound if any\n    lower: Optional[float] = None\n\n    # upper bound if any\n    upper: Optional[float] = None\n\n    # initial value\n    # default to the middle point if completely bounded\n    init: Optional[float] = None\n\n    # step size for an update\n    # defaults to 1 if unbounded\n    # or 1/6 of the range if completely bounded\n    step: Optional[float] = None\n\n    # cast to integer\n    integer: bool = False\n\n    # logarithmically distributed\n    log: bool = False\n\n\n@dataclass\nclass OptimConf:\n\n    # name of the Nevergrad optimizer to use. Here is a sample:\n    #   - \"OnePlusOne\" extremely simple and robust, especially at low budget, but\n    #     tends to converge early.\n    #   - \"CMA\" very good algorithm, but may require a significant budget (> 120)\n    #   - \"TwoPointsDE\": an algorithm good in a wide range of settings, for significant\n    #     budgets (> 120).\n    #   - \"NGOpt\" an algorithm aiming at identifying the best optimizer given your input\n    #     definition (updated regularly)\n    # find out more within nevergrad's documentation:\n    # https://github.com/facebookresearch/nevergrad/\n    optimizer: str = \"NGOpt\"\n\n    # total number of function evaluations to perform\n    budget: int = 80\n\n    # number of parallel workers for performing function evaluations\n    num_workers: int = 10\n\n    # set to true if the function evaluations are noisy\n    noisy: bool = False\n\n    # set to true for performing maximization instead of minimization\n    maximize: bool = False\n\n    # optimization seed, for reproducibility\n    seed: Optional[int] = None\n\n\n@dataclass\nclass NevergradSweeperConf:\n    _target_: str = (\n        \"hydra_plugins.hydra_nevergrad_sweeper.nevergrad_sweeper.NevergradSweeper\"\n    )\n\n    # configuration of the optimizer\n    optim: OptimConf = OptimConf()\n\n    # default parametrization of the search space\n    # can be specified:\n    # - as a string, like commandline arguments\n    # - as a list, for categorical variables\n    # - as a full scalar specification\n    parametrization: Dict[str, Any] = field(default_factory=dict)\n\n\nConfigStore.instance().store(\n    group=\"hydra/sweeper\",\n    name=\"nevergrad\",\n    node=NevergradSweeperConf,\n    provider=\"nevergrad\",\n)\n", "path": "plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py"}, {"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\nfrom typing import (\n    Any,\n    Dict,\n    List,\n    MutableMapping,\n    MutableSequence,\n    Optional,\n    Tuple,\n    Union,\n)\n\nimport nevergrad as ng\nfrom hydra.core.override_parser.overrides_parser import OverridesParser\nfrom hydra.core.override_parser.types import (\n    ChoiceSweep,\n    IntervalSweep,\n    Override,\n    Transformer,\n)\nfrom hydra.core.plugins import Plugins\nfrom hydra.plugins.launcher import Launcher\nfrom hydra.plugins.sweeper import Sweeper\nfrom hydra.types import HydraContext, TaskFunction\nfrom omegaconf import DictConfig, ListConfig, OmegaConf\n\nfrom .config import OptimConf, ScalarConfigSpec\n\nlog = logging.getLogger(__name__)\n\n\ndef create_nevergrad_param_from_config(\n    config: Union[MutableSequence[Any], MutableMapping[str, Any]]\n) -> Any:\n    if isinstance(config, MutableSequence):\n        if isinstance(config, ListConfig):\n            config = OmegaConf.to_container(config, resolve=True)  # type: ignore\n        return ng.p.Choice(config)\n    if isinstance(config, MutableMapping):\n        specs = ScalarConfigSpec(**config)\n        init = [\"init\", \"lower\", \"upper\"]\n        init_params = {x: getattr(specs, x) for x in init}\n        if not specs.log:\n            scalar = ng.p.Scalar(**init_params)\n            if specs.step is not None:\n                scalar.set_mutation(sigma=specs.step)\n        else:\n            if specs.step is not None:\n                init_params[\"exponent\"] = specs.step\n            scalar = ng.p.Log(**init_params)\n        if specs.integer:\n            scalar.set_integer_casting()\n        return scalar\n    return config\n\n\ndef create_nevergrad_parameter_from_override(override: Override) -> Any:\n    val = override.value()\n    if not override.is_sweep_override():\n        return val\n    if override.is_choice_sweep():\n        assert isinstance(val, ChoiceSweep)\n        vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]\n        if \"ordered\" in val.tags:\n            return ng.p.TransitionChoice(vals)\n        else:\n            return ng.p.Choice(vals)\n    elif override.is_range_sweep():\n        vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]\n        return ng.p.Choice(vals)\n    elif override.is_interval_sweep():\n        assert isinstance(val, IntervalSweep)\n        if \"log\" in val.tags:\n            scalar = ng.p.Log(lower=val.start, upper=val.end)\n        else:\n            scalar = ng.p.Scalar(lower=val.start, upper=val.end)  # type: ignore\n        if isinstance(val.start, int):\n            scalar.set_integer_casting()\n        return scalar\n\n\nclass NevergradSweeperImpl(Sweeper):\n    def __init__(\n        self,\n        optim: OptimConf,\n        parametrization: Optional[DictConfig],\n    ):\n        self.opt_config = optim\n        self.config: Optional[DictConfig] = None\n        self.launcher: Optional[Launcher] = None\n        self.hydra_context: Optional[HydraContext] = None\n        self.job_results = None\n        self.parametrization: Dict[str, Any] = {}\n        if parametrization is not None:\n            assert isinstance(parametrization, DictConfig)\n            self.parametrization = {\n                str(x): create_nevergrad_param_from_config(y)\n                for x, y in parametrization.items()\n            }\n        self.job_idx: Optional[int] = None\n\n    def setup(\n        self,\n        *,\n        hydra_context: HydraContext,\n        task_function: TaskFunction,\n        config: DictConfig,\n    ) -> None:\n        self.job_idx = 0\n        self.config = config\n        self.hydra_context = hydra_context\n        self.launcher = Plugins.instance().instantiate_launcher(\n            hydra_context=hydra_context, task_function=task_function, config=config\n        )\n\n    def sweep(self, arguments: List[str]) -> None:\n\n        assert self.config is not None\n        assert self.launcher is not None\n        assert self.job_idx is not None\n        direction = -1 if self.opt_config.maximize else 1\n        name = \"maximization\" if self.opt_config.maximize else \"minimization\"\n        # Override the parametrization from commandline\n        params = dict(self.parametrization)\n\n        parser = OverridesParser.create()\n        parsed = parser.parse_overrides(arguments)\n\n        for override in parsed:\n            params[\n                override.get_key_element()\n            ] = create_nevergrad_parameter_from_override(override)\n\n        parametrization = ng.p.Dict(**params)\n        parametrization.function.deterministic = not self.opt_config.noisy\n        parametrization.random_state.seed(self.opt_config.seed)\n        # log and build the optimizer\n        opt = self.opt_config.optimizer\n        remaining_budget = self.opt_config.budget\n        nw = self.opt_config.num_workers\n        log.info(\n            f\"NevergradSweeper(optimizer={opt}, budget={remaining_budget}, \"\n            f\"num_workers={nw}) {name}\"\n        )\n        log.info(f\"with parametrization {parametrization}\")\n        log.info(f\"Sweep output dir: {self.config.hydra.sweep.dir}\")\n        optimizer = ng.optimizers.registry[opt](parametrization, remaining_budget, nw)\n        # loop!\n        all_returns: List[Any] = []\n        best: Tuple[float, ng.p.Parameter] = (float(\"inf\"), parametrization)\n        while remaining_budget > 0:\n            batch = min(nw, remaining_budget)\n            remaining_budget -= batch\n            candidates = [optimizer.ask() for _ in range(batch)]\n            overrides = list(\n                tuple(f\"{x}={y}\" for x, y in c.value.items()) for c in candidates\n            )\n            self.validate_batch_is_legal(overrides)\n            returns = self.launcher.launch(overrides, initial_job_idx=self.job_idx)\n            self.job_idx += len(returns)\n            # would have been nice to avoid waiting for all jobs to finish\n            # aka batch size Vs steady state (launching a new job whenever one is done)\n            for cand, ret in zip(candidates, returns):\n                loss = direction * ret.return_value\n                optimizer.tell(cand, loss)\n                if loss < best[0]:\n                    best = (loss, cand)\n            all_returns.extend(returns)\n        recom = optimizer.provide_recommendation()\n        results_to_serialize = {\n            \"name\": \"nevergrad\",\n            \"best_evaluated_params\": best[1].value,\n            \"best_evaluated_result\": direction * best[0],\n        }\n        OmegaConf.save(\n            OmegaConf.create(results_to_serialize),\n            f\"{self.config.hydra.sweep.dir}/optimization_results.yaml\",\n        )\n        log.info(\n            \"Best parameters: %s\", \" \".join(f\"{x}={y}\" for x, y in recom.value.items())\n        )\n", "path": "plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py"}, {"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\n\nimport hydra\nfrom omegaconf import DictConfig\n\nlog = logging.getLogger(__name__)\n\n\n@hydra.main(config_path=\".\", config_name=\"config\")\ndef dummy_training(cfg: DictConfig) -> float:\n    \"\"\"A dummy function to minimize\n    Minimum is 0.0 at:\n    lr = 0.12, dropout=0.33, db=mnist, batch_size=4\n    \"\"\"\n    do = cfg.dropout\n    bs = cfg.batch_size\n    out = float(\n        abs(do - 0.33) + int(cfg.db == \"mnist\") + abs(cfg.lr - 0.12) + abs(bs - 4)\n    )\n    log.info(\n        f\"dummy_training(dropout={do:.3f}, lr={cfg.lr:.3f}, db={cfg.db}, batch_size={bs}) = {out:.3f}\",\n    )\n    return out\n\n\nif __name__ == \"__main__\":\n    dummy_training()\n", "path": "plugins/hydra_nevergrad_sweeper/example/my_app.py"}], "after_files": [{"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, Optional\n\nfrom hydra.core.config_store import ConfigStore\n\n\n@dataclass\nclass ScalarConfigSpec:\n    \"\"\"Representation of all the options to define\n    a scalar.\n    \"\"\"\n\n    # lower bound if any\n    lower: Optional[float] = None\n\n    # upper bound if any\n    upper: Optional[float] = None\n\n    # initial value\n    # default to the middle point if completely bounded\n    init: Optional[float] = None\n\n    # step size for an update\n    # defaults to 1 if unbounded\n    # or 1/6 of the range if completely bounded\n    step: Optional[float] = None\n\n    # cast to integer\n    integer: bool = False\n\n    # logarithmically distributed\n    log: bool = False\n\n\n@dataclass\nclass OptimConf:\n\n    # name of the Nevergrad optimizer to use. Here is a sample:\n    #   - \"OnePlusOne\" extremely simple and robust, especially at low budget, but\n    #     tends to converge early.\n    #   - \"CMA\" very good algorithm, but may require a significant budget (> 120)\n    #   - \"TwoPointsDE\": an algorithm good in a wide range of settings, for significant\n    #     budgets (> 120).\n    #   - \"NGOpt\" an algorithm aiming at identifying the best optimizer given your input\n    #     definition (updated regularly)\n    # find out more within nevergrad's documentation:\n    # https://github.com/facebookresearch/nevergrad/\n    optimizer: str = \"NGOpt\"\n\n    # total number of function evaluations to perform\n    budget: int = 80\n\n    # number of parallel workers for performing function evaluations\n    num_workers: int = 10\n\n    # set to true if the function evaluations are noisy\n    noisy: bool = False\n\n    # set to true for performing maximization instead of minimization\n    maximize: bool = False\n\n    # optimization seed, for reproducibility\n    seed: Optional[int] = None\n\n    # maximum authorized failure rate for a batch of parameters\n    max_failure_rate: float = 0.0\n\n\n@dataclass\nclass NevergradSweeperConf:\n    _target_: str = (\n        \"hydra_plugins.hydra_nevergrad_sweeper.nevergrad_sweeper.NevergradSweeper\"\n    )\n\n    # configuration of the optimizer\n    optim: OptimConf = OptimConf()\n\n    # default parametrization of the search space\n    # can be specified:\n    # - as a string, like commandline arguments\n    # - as a list, for categorical variables\n    # - as a full scalar specification\n    parametrization: Dict[str, Any] = field(default_factory=dict)\n\n\nConfigStore.instance().store(\n    group=\"hydra/sweeper\",\n    name=\"nevergrad\",\n    node=NevergradSweeperConf,\n    provider=\"nevergrad\",\n)\n", "path": "plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py"}, {"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\nimport math\nfrom typing import (\n    Any,\n    Dict,\n    List,\n    MutableMapping,\n    MutableSequence,\n    Optional,\n    Tuple,\n    Union,\n)\n\nimport nevergrad as ng\nfrom hydra.core import utils\nfrom hydra.core.override_parser.overrides_parser import OverridesParser\nfrom hydra.core.override_parser.types import (\n    ChoiceSweep,\n    IntervalSweep,\n    Override,\n    Transformer,\n)\nfrom hydra.core.plugins import Plugins\nfrom hydra.plugins.launcher import Launcher\nfrom hydra.plugins.sweeper import Sweeper\nfrom hydra.types import HydraContext, TaskFunction\nfrom omegaconf import DictConfig, ListConfig, OmegaConf\n\nfrom .config import OptimConf, ScalarConfigSpec\n\nlog = logging.getLogger(__name__)\n\n\ndef create_nevergrad_param_from_config(\n    config: Union[MutableSequence[Any], MutableMapping[str, Any]]\n) -> Any:\n    if isinstance(config, MutableSequence):\n        if isinstance(config, ListConfig):\n            config = OmegaConf.to_container(config, resolve=True)  # type: ignore\n        return ng.p.Choice(config)\n    if isinstance(config, MutableMapping):\n        specs = ScalarConfigSpec(**config)\n        init = [\"init\", \"lower\", \"upper\"]\n        init_params = {x: getattr(specs, x) for x in init}\n        if not specs.log:\n            scalar = ng.p.Scalar(**init_params)\n            if specs.step is not None:\n                scalar.set_mutation(sigma=specs.step)\n        else:\n            if specs.step is not None:\n                init_params[\"exponent\"] = specs.step\n            scalar = ng.p.Log(**init_params)\n        if specs.integer:\n            scalar.set_integer_casting()\n        return scalar\n    return config\n\n\ndef create_nevergrad_parameter_from_override(override: Override) -> Any:\n    val = override.value()\n    if not override.is_sweep_override():\n        return val\n    if override.is_choice_sweep():\n        assert isinstance(val, ChoiceSweep)\n        vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]\n        if \"ordered\" in val.tags:\n            return ng.p.TransitionChoice(vals)\n        else:\n            return ng.p.Choice(vals)\n    elif override.is_range_sweep():\n        vals = [x for x in override.sweep_iterator(transformer=Transformer.encode)]\n        return ng.p.Choice(vals)\n    elif override.is_interval_sweep():\n        assert isinstance(val, IntervalSweep)\n        if \"log\" in val.tags:\n            scalar = ng.p.Log(lower=val.start, upper=val.end)\n        else:\n            scalar = ng.p.Scalar(lower=val.start, upper=val.end)  # type: ignore\n        if isinstance(val.start, int):\n            scalar.set_integer_casting()\n        return scalar\n\n\nclass NevergradSweeperImpl(Sweeper):\n    def __init__(\n        self,\n        optim: OptimConf,\n        parametrization: Optional[DictConfig],\n    ):\n        self.opt_config = optim\n        self.config: Optional[DictConfig] = None\n        self.launcher: Optional[Launcher] = None\n        self.hydra_context: Optional[HydraContext] = None\n        self.job_results = None\n        self.parametrization: Dict[str, Any] = {}\n        if parametrization is not None:\n            assert isinstance(parametrization, DictConfig)\n            self.parametrization = {\n                str(x): create_nevergrad_param_from_config(y)\n                for x, y in parametrization.items()\n            }\n        self.job_idx: Optional[int] = None\n\n    def setup(\n        self,\n        *,\n        hydra_context: HydraContext,\n        task_function: TaskFunction,\n        config: DictConfig,\n    ) -> None:\n        self.job_idx = 0\n        self.config = config\n        self.hydra_context = hydra_context\n        self.launcher = Plugins.instance().instantiate_launcher(\n            hydra_context=hydra_context, task_function=task_function, config=config\n        )\n\n    def sweep(self, arguments: List[str]) -> None:\n\n        assert self.config is not None\n        assert self.launcher is not None\n        assert self.job_idx is not None\n        direction = -1 if self.opt_config.maximize else 1\n        name = \"maximization\" if self.opt_config.maximize else \"minimization\"\n        # Override the parametrization from commandline\n        params = dict(self.parametrization)\n\n        parser = OverridesParser.create()\n        parsed = parser.parse_overrides(arguments)\n\n        for override in parsed:\n            params[\n                override.get_key_element()\n            ] = create_nevergrad_parameter_from_override(override)\n\n        parametrization = ng.p.Dict(**params)\n        parametrization.function.deterministic = not self.opt_config.noisy\n        parametrization.random_state.seed(self.opt_config.seed)\n        # log and build the optimizer\n        opt = self.opt_config.optimizer\n        remaining_budget = self.opt_config.budget\n        nw = self.opt_config.num_workers\n        log.info(\n            f\"NevergradSweeper(optimizer={opt}, budget={remaining_budget}, \"\n            f\"num_workers={nw}) {name}\"\n        )\n        log.info(f\"with parametrization {parametrization}\")\n        log.info(f\"Sweep output dir: {self.config.hydra.sweep.dir}\")\n        optimizer = ng.optimizers.registry[opt](parametrization, remaining_budget, nw)\n        # loop!\n        all_returns: List[Any] = []\n        best: Tuple[float, ng.p.Parameter] = (float(\"inf\"), parametrization)\n        while remaining_budget > 0:\n            batch = min(nw, remaining_budget)\n            remaining_budget -= batch\n            candidates = [optimizer.ask() for _ in range(batch)]\n            overrides = list(\n                tuple(f\"{x}={y}\" for x, y in c.value.items()) for c in candidates\n            )\n            self.validate_batch_is_legal(overrides)\n            returns = self.launcher.launch(overrides, initial_job_idx=self.job_idx)\n            # would have been nice to avoid waiting for all jobs to finish\n            # aka batch size Vs steady state (launching a new job whenever one is done)\n            self.job_idx += len(returns)\n            # check job status and prepare losses\n            failures = 0\n            for cand, ret in zip(candidates, returns):\n                if ret.status == utils.JobStatus.COMPLETED:\n                    rectified_loss = direction * ret.return_value\n                else:\n                    rectified_loss = math.inf\n                    failures += 1\n                    try:\n                        ret.return_value\n                    except Exception as e:\n                        log.warning(f\"Returning infinity for failed experiment: {e}\")\n                optimizer.tell(cand, rectified_loss)\n                if rectified_loss < best[0]:\n                    best = (rectified_loss, cand)\n            # raise if too many failures\n            if failures / len(returns) > self.opt_config.max_failure_rate:\n                log.error(\n                    f\"Failed {failures} times out of {len(returns)} \"\n                    f\"with max_failure_rate={self.opt_config.max_failure_rate}\"\n                )\n                for ret in returns:\n                    ret.return_value  # delegate raising to JobReturn, with actual traceback\n            all_returns.extend(returns)\n        recom = optimizer.provide_recommendation()\n        results_to_serialize = {\n            \"name\": \"nevergrad\",\n            \"best_evaluated_params\": best[1].value,\n            \"best_evaluated_result\": direction * best[0],\n        }\n        OmegaConf.save(\n            OmegaConf.create(results_to_serialize),\n            f\"{self.config.hydra.sweep.dir}/optimization_results.yaml\",\n        )\n        log.info(\n            \"Best parameters: %s\", \" \".join(f\"{x}={y}\" for x, y in recom.value.items())\n        )\n", "path": "plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py"}, {"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\n\nimport hydra\nfrom omegaconf import DictConfig\n\nlog = logging.getLogger(__name__)\n\n\n@hydra.main(config_path=\".\", config_name=\"config\")\ndef dummy_training(cfg: DictConfig) -> float:\n    \"\"\"A dummy function to minimize\n    Minimum is 0.0 at:\n    lr = 0.12, dropout=0.33, db=mnist, batch_size=4\n    \"\"\"\n    do = cfg.dropout\n    bs = cfg.batch_size\n    out = float(\n        abs(do - 0.33) + int(cfg.db == \"mnist\") + abs(cfg.lr - 0.12) + abs(bs - 4)\n    )\n    log.info(\n        f\"dummy_training(dropout={do:.3f}, lr={cfg.lr:.3f}, db={cfg.db}, batch_size={bs}) = {out:.3f}\",\n    )\n    if cfg.error:\n        raise RuntimeError(\"cfg.error is True\")\n    return out\n\n\nif __name__ == \"__main__\":\n    dummy_training()\n", "path": "plugins/hydra_nevergrad_sweeper/example/my_app.py"}]}