We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[rllib] incorrect model output for DQN with torch and dueling=false 
<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->

### What is the problem?

The output fo the DQN model is not within the action space. 

Something is wrong when constructing the torch model when dueling is off. The output dimension of the model is equal to whatever is passed in "fcnet_hiddens" instead of being of the size of the action space. 

*Ray version and other system information (Python version, TensorFlow version, OS):*
- ray==0.9.0.dev0 
- python 3.6.10
- mac OS

### Reproduction (REQUIRED)

```python
import ray
from ray import tune

ray.init()

config = {
    "env": "CartPole-v1",
    "num_workers": 1,
    "train_batch_size": 128,
    "learning_starts": 128,
    "model": {"fcnet_hiddens": [32]},
    "dueling": False ,
    "framework": "torch"
}

tune.run("DQN", name="MWE", config=config, stop={"training_iteration": 100})
```

- [x] I have verified my script runs in a clean environment and reproduces the issue.
- [x] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `rllib/agents/dqn/dqn_torch_model.py`
Content:
```
1 import numpy as np
2 
3 from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
4 from ray.rllib.utils.framework import try_import_torch
5 
6 torch, nn = try_import_torch()
7 
8 
9 class DQNTorchModel(TorchModelV2, nn.Module):
10     """Extension of standard TorchModelV2 to provide dueling-Q functionality.
11     """
12 
13     def __init__(
14             self,
15             obs_space,
16             action_space,
17             num_outputs,
18             model_config,
19             name,
20             *,
21             dueling=False,
22             q_hiddens=(256, ),
23             dueling_activation="relu",
24             use_noisy=False,
25             sigma0=0.5,
26             # TODO(sven): Move `add_layer_norm` into ModelCatalog as
27             #  generic option, then error if we use ParameterNoise as
28             #  Exploration type and do not have any LayerNorm layers in
29             #  the net.
30             add_layer_norm=False):
31         """Initialize variables of this model.
32 
33         Extra model kwargs:
34             dueling (bool): Whether to build the advantage(A)/value(V) heads
35                 for DDQN. If True, Q-values are calculated as:
36                 Q = (A - mean[A]) + V. If False, raw NN output is interpreted
37                 as Q-values.
38             q_hiddens (List[int]): List of layer-sizes after(!) the
39                 Advantages(A)/Value(V)-split. Hence, each of the A- and V-
40                 branches will have this structure of Dense layers. To define
41                 the NN before this A/V-split, use - as always -
42                 config["model"]["fcnet_hiddens"].
43             dueling_activation (str): The activation to use for all dueling
44                 layers (A- and V-branch). One of "relu", "tanh", "linear".
45             use_noisy (bool): use noisy nets
46             sigma0 (float): initial value of noisy nets
47             add_layer_norm (bool): Enable layer norm (for param noise).
48         """
49         nn.Module.__init__(self)
50         super(DQNTorchModel, self).__init__(obs_space, action_space,
51                                             num_outputs, model_config, name)
52 
53         self.dueling = dueling
54         ins = num_outputs
55 
56         # Dueling case: Build the shared (advantages and value) fc-network.
57         advantage_module = nn.Sequential()
58         value_module = None
59         if self.dueling:
60             value_module = nn.Sequential()
61             for i, n in enumerate(q_hiddens):
62                 advantage_module.add_module("dueling_A_{}".format(i),
63                                             nn.Linear(ins, n))
64                 value_module.add_module("dueling_V_{}".format(i),
65                                         nn.Linear(ins, n))
66                 # Add activations if necessary.
67                 if dueling_activation == "relu":
68                     advantage_module.add_module("dueling_A_act_{}".format(i),
69                                                 nn.ReLU())
70                     value_module.add_module("dueling_V_act_{}".format(i),
71                                             nn.ReLU())
72                 elif dueling_activation == "tanh":
73                     advantage_module.add_module("dueling_A_act_{}".format(i),
74                                                 nn.Tanh())
75                     value_module.add_module("dueling_V_act_{}".format(i),
76                                             nn.Tanh())
77 
78                 # Add LayerNorm after each Dense.
79                 if add_layer_norm:
80                     advantage_module.add_module("LayerNorm_A_{}".format(i),
81                                                 nn.LayerNorm(n))
82                     value_module.add_module("LayerNorm_V_{}".format(i),
83                                             nn.LayerNorm(n))
84                 ins = n
85             # Actual Advantages layer (nodes=num-actions) and
86             # value layer (nodes=1).
87             advantage_module.add_module("A", nn.Linear(ins, action_space.n))
88             value_module.add_module("V", nn.Linear(ins, 1))
89         # Non-dueling:
90         # Q-value layer (use main module's outputs as Q-values).
91         else:
92             pass
93 
94         self.advantage_module = advantage_module
95         self.value_module = value_module
96 
97     def get_advantages_or_q_values(self, model_out):
98         """Returns distributional values for Q(s, a) given a state embedding.
99 
100         Override this in your custom model to customize the Q output head.
101 
102         Arguments:
103             model_out (Tensor): embedding from the model layers
104 
105         Returns:
106             (action_scores, logits, dist) if num_atoms == 1, otherwise
107             (action_scores, z, support_logits_per_action, logits, dist)
108         """
109 
110         return self.advantage_module(model_out)
111 
112     def get_state_value(self, model_out):
113         """Returns the state value prediction for the given state embedding."""
114 
115         return self.value_module(model_out)
116 
117     def _noisy_layer(self, action_in, out_size, sigma0, non_linear=True):
118         """
119         a common dense layer: y = w^{T}x + b
120         a noisy layer: y = (w + \\epsilon_w*\\sigma_w)^{T}x +
121             (b+\\epsilon_b*\\sigma_b)
122         where \epsilon are random variables sampled from factorized normal
123         distributions and \\sigma are trainable variables which are expected to
124         vanish along the training procedure
125         """
126         in_size = int(action_in.shape[1])
127 
128         epsilon_in = torch.normal(
129             mean=torch.zeros([in_size]), std=torch.ones([in_size]))
130         epsilon_out = torch.normal(
131             mean=torch.zeros([out_size]), std=torch.ones([out_size]))
132         epsilon_in = self._f_epsilon(epsilon_in)
133         epsilon_out = self._f_epsilon(epsilon_out)
134         epsilon_w = torch.matmul(
135             torch.unsqueeze(epsilon_in, -1),
136             other=torch.unsqueeze(epsilon_out, 0))
137         epsilon_b = epsilon_out
138 
139         sigma_w = torch.Tensor(
140             data=np.random.uniform(
141                 low=-1.0 / np.sqrt(float(in_size)),
142                 high=1.0 / np.sqrt(float(in_size)),
143                 size=[in_size, out_size]),
144             dtype=torch.float32,
145             requires_grad=True)
146         # TF noise generation can be unreliable on GPU
147         # If generating the noise on the CPU,
148         # lowering sigma0 to 0.1 may be helpful
149         sigma_b = torch.Tensor(
150             data=np.full(
151                 shape=[out_size], fill_value=sigma0 / np.sqrt(float(in_size))),
152             requires_grad=True)
153         w = torch.Tensor(
154             data=np.full(
155                 shape=[in_size, out_size],
156                 fill_value=6 / np.sqrt(float(in_size) + float(out_size))),
157             requires_grad=True)
158         b = torch.Tensor(data=np.zeros([out_size]), requires_grad=True)
159         action_activation = torch.matmul(action_in, w + sigma_w * epsilon_w) \
160             + b + sigma_b * epsilon_b
161 
162         if not non_linear:
163             return action_activation
164         return nn.functional.relu(action_activation)
165 
166     def _f_epsilon(self, x):
167         return torch.sign(x) * torch.pow(torch.abs(x), 0.5)
168
```

Path: `rllib/examples/parametric_actions_cartpole.py`
Content:
```
1 """Example of handling variable length and/or parametric action spaces.
2 
3 This is a toy example of the action-embedding based approach for handling large
4 discrete action spaces (potentially infinite in size), similar to this:
5 
6     https://neuro.cs.ut.ee/the-use-of-embeddings-in-openai-five/
7 
8 This currently works with RLlib's policy gradient style algorithms
9 (e.g., PG, PPO, IMPALA, A2C) and also DQN.
10 
11 Note that since the model outputs now include "-inf" tf.float32.min
12 values, not all algorithm options are supported at the moment. For example,
13 algorithms might crash if they don't properly ignore the -inf action scores.
14 Working configurations are given below.
15 """
16 
17 import argparse
18 
19 import ray
20 from ray import tune
21 from ray.rllib.examples.env.parametric_actions_cartpole import \
22     ParametricActionsCartPole
23 from ray.rllib.examples.models.parametric_actions_model import \
24     ParametricActionsModel, TorchParametricActionsModel
25 from ray.rllib.models import ModelCatalog
26 from ray.rllib.utils.test_utils import check_learning_achieved
27 from ray.tune.registry import register_env
28 
29 parser = argparse.ArgumentParser()
30 parser.add_argument("--run", type=str, default="PPO")
31 parser.add_argument("--torch", action="store_true")
32 parser.add_argument("--as-test", action="store_true")
33 parser.add_argument("--stop-iters", type=int, default=200)
34 parser.add_argument("--stop-reward", type=float, default=150.0)
35 parser.add_argument("--stop-timesteps", type=int, default=100000)
36 
37 if __name__ == "__main__":
38     args = parser.parse_args()
39     ray.init()
40 
41     register_env("pa_cartpole", lambda _: ParametricActionsCartPole(10))
42     ModelCatalog.register_custom_model(
43         "pa_model", TorchParametricActionsModel
44         if args.torch else ParametricActionsModel)
45 
46     if args.run == "DQN":
47         cfg = {
48             # TODO(ekl) we need to set these to prevent the masked values
49             # from being further processed in DistributionalQModel, which
50             # would mess up the masking. It is possible to support these if we
51             # defined a a custom DistributionalQModel that is aware of masking.
52             "hiddens": [],
53             "dueling": False,
54         }
55     else:
56         cfg = {}
57 
58     config = dict({
59         "env": "pa_cartpole",
60         "model": {
61             "custom_model": "pa_model",
62         },
63         "num_workers": 0,
64         "framework": "torch" if args.torch else "tf",
65     }, **cfg)
66 
67     stop = {
68         "training_iteration": args.stop_iters,
69         "timesteps_total": args.stop_timesteps,
70         "episode_reward_mean": args.stop_reward,
71     }
72 
73     results = tune.run(args.run, stop=stop, config=config, verbose=1)
74 
75     if args.as_test:
76         check_learning_achieved(results, args.stop_reward)
77 
78     ray.shutdown()
79
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/rllib/agents/dqn/dqn_torch_model.py b/rllib/agents/dqn/dqn_torch_model.py
--- a/rllib/agents/dqn/dqn_torch_model.py
+++ b/rllib/agents/dqn/dqn_torch_model.py
@@ -53,46 +53,45 @@
         self.dueling = dueling
         ins = num_outputs
 
-        # Dueling case: Build the shared (advantages and value) fc-network.
         advantage_module = nn.Sequential()
-        value_module = None
-        if self.dueling:
-            value_module = nn.Sequential()
-            for i, n in enumerate(q_hiddens):
-                advantage_module.add_module("dueling_A_{}".format(i),
-                                            nn.Linear(ins, n))
-                value_module.add_module("dueling_V_{}".format(i),
+        value_module = nn.Sequential()
+
+        # Dueling case: Build the shared (advantages and value) fc-network.
+        for i, n in enumerate(q_hiddens):
+            advantage_module.add_module("dueling_A_{}".format(i),
                                         nn.Linear(ins, n))
-                # Add activations if necessary.
-                if dueling_activation == "relu":
-                    advantage_module.add_module("dueling_A_act_{}".format(i),
-                                                nn.ReLU())
-                    value_module.add_module("dueling_V_act_{}".format(i),
+            value_module.add_module("dueling_V_{}".format(i),
+                                    nn.Linear(ins, n))
+            # Add activations if necessary.
+            if dueling_activation == "relu":
+                advantage_module.add_module("dueling_A_act_{}".format(i),
                                             nn.ReLU())
-                elif dueling_activation == "tanh":
-                    advantage_module.add_module("dueling_A_act_{}".format(i),
-                                                nn.Tanh())
-                    value_module.add_module("dueling_V_act_{}".format(i),
+                value_module.add_module("dueling_V_act_{}".format(i),
+                                        nn.ReLU())
+            elif dueling_activation == "tanh":
+                advantage_module.add_module("dueling_A_act_{}".format(i),
                                             nn.Tanh())
+                value_module.add_module("dueling_V_act_{}".format(i),
+                                        nn.Tanh())
 
-                # Add LayerNorm after each Dense.
-                if add_layer_norm:
-                    advantage_module.add_module("LayerNorm_A_{}".format(i),
-                                                nn.LayerNorm(n))
-                    value_module.add_module("LayerNorm_V_{}".format(i),
+            # Add LayerNorm after each Dense.
+            if add_layer_norm:
+                advantage_module.add_module("LayerNorm_A_{}".format(i),
                                             nn.LayerNorm(n))
-                ins = n
-            # Actual Advantages layer (nodes=num-actions) and
-            # value layer (nodes=1).
+                value_module.add_module("LayerNorm_V_{}".format(i),
+                                        nn.LayerNorm(n))
+            ins = n
+
+        # Actual Advantages layer (nodes=num-actions).
+        if q_hiddens:
             advantage_module.add_module("A", nn.Linear(ins, action_space.n))
-            value_module.add_module("V", nn.Linear(ins, 1))
-        # Non-dueling:
-        # Q-value layer (use main module's outputs as Q-values).
-        else:
-            pass
 
         self.advantage_module = advantage_module
-        self.value_module = value_module
+
+        # Value layer (nodes=1).
+        if self.dueling:
+            value_module.add_module("V", nn.Linear(ins, 1))
+            self.value_module = value_module
 
     def get_advantages_or_q_values(self, model_out):
         """Returns distributional values for Q(s, a) given a state embedding.
diff --git a/rllib/examples/parametric_actions_cartpole.py b/rllib/examples/parametric_actions_cartpole.py
--- a/rllib/examples/parametric_actions_cartpole.py
+++ b/rllib/examples/parametric_actions_cartpole.py
@@ -48,7 +48,7 @@
             # TODO(ekl) we need to set these to prevent the masked values
             # from being further processed in DistributionalQModel, which
             # would mess up the masking. It is possible to support these if we
-            # defined a a custom DistributionalQModel that is aware of masking.
+            # defined a custom DistributionalQModel that is aware of masking.
             "hiddens": [],
             "dueling": False,
         }

{"golden_diff": "diff --git a/rllib/agents/dqn/dqn_torch_model.py b/rllib/agents/dqn/dqn_torch_model.py\n--- a/rllib/agents/dqn/dqn_torch_model.py\n+++ b/rllib/agents/dqn/dqn_torch_model.py\n@@ -53,46 +53,45 @@\n         self.dueling = dueling\n         ins = num_outputs\n \n-        # Dueling case: Build the shared (advantages and value) fc-network.\n         advantage_module = nn.Sequential()\n-        value_module = None\n-        if self.dueling:\n-            value_module = nn.Sequential()\n-            for i, n in enumerate(q_hiddens):\n-                advantage_module.add_module(\"dueling_A_{}\".format(i),\n-                                            nn.Linear(ins, n))\n-                value_module.add_module(\"dueling_V_{}\".format(i),\n+        value_module = nn.Sequential()\n+\n+        # Dueling case: Build the shared (advantages and value) fc-network.\n+        for i, n in enumerate(q_hiddens):\n+            advantage_module.add_module(\"dueling_A_{}\".format(i),\n                                         nn.Linear(ins, n))\n-                # Add activations if necessary.\n-                if dueling_activation == \"relu\":\n-                    advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n-                                                nn.ReLU())\n-                    value_module.add_module(\"dueling_V_act_{}\".format(i),\n+            value_module.add_module(\"dueling_V_{}\".format(i),\n+                                    nn.Linear(ins, n))\n+            # Add activations if necessary.\n+            if dueling_activation == \"relu\":\n+                advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n                                             nn.ReLU())\n-                elif dueling_activation == \"tanh\":\n-                    advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n-                                                nn.Tanh())\n-                    value_module.add_module(\"dueling_V_act_{}\".format(i),\n+                value_module.add_module(\"dueling_V_act_{}\".format(i),\n+                                        nn.ReLU())\n+            elif dueling_activation == \"tanh\":\n+                advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n                                             nn.Tanh())\n+                value_module.add_module(\"dueling_V_act_{}\".format(i),\n+                                        nn.Tanh())\n \n-                # Add LayerNorm after each Dense.\n-                if add_layer_norm:\n-                    advantage_module.add_module(\"LayerNorm_A_{}\".format(i),\n-                                                nn.LayerNorm(n))\n-                    value_module.add_module(\"LayerNorm_V_{}\".format(i),\n+            # Add LayerNorm after each Dense.\n+            if add_layer_norm:\n+                advantage_module.add_module(\"LayerNorm_A_{}\".format(i),\n                                             nn.LayerNorm(n))\n-                ins = n\n-            # Actual Advantages layer (nodes=num-actions) and\n-            # value layer (nodes=1).\n+                value_module.add_module(\"LayerNorm_V_{}\".format(i),\n+                                        nn.LayerNorm(n))\n+            ins = n\n+\n+        # Actual Advantages layer (nodes=num-actions).\n+        if q_hiddens:\n             advantage_module.add_module(\"A\", nn.Linear(ins, action_space.n))\n-            value_module.add_module(\"V\", nn.Linear(ins, 1))\n-        # Non-dueling:\n-        # Q-value layer (use main module's outputs as Q-values).\n-        else:\n-            pass\n \n         self.advantage_module = advantage_module\n-        self.value_module = value_module\n+\n+        # Value layer (nodes=1).\n+        if self.dueling:\n+            value_module.add_module(\"V\", nn.Linear(ins, 1))\n+            self.value_module = value_module\n \n     def get_advantages_or_q_values(self, model_out):\n         \"\"\"Returns distributional values for Q(s, a) given a state embedding.\ndiff --git a/rllib/examples/parametric_actions_cartpole.py b/rllib/examples/parametric_actions_cartpole.py\n--- a/rllib/examples/parametric_actions_cartpole.py\n+++ b/rllib/examples/parametric_actions_cartpole.py\n@@ -48,7 +48,7 @@\n             # TODO(ekl) we need to set these to prevent the masked values\n             # from being further processed in DistributionalQModel, which\n             # would mess up the masking. It is possible to support these if we\n-            # defined a a custom DistributionalQModel that is aware of masking.\n+            # defined a custom DistributionalQModel that is aware of masking.\n             \"hiddens\": [],\n             \"dueling\": False,\n         }\n", "issue": "[rllib] incorrect model output for DQN with torch and dueling=false \n<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->\r\n\r\n### What is the problem?\r\n\r\nThe output fo the DQN model is not within the action space. \r\n\r\nSomething is wrong when constructing the torch model when dueling is off. The output dimension of the model is equal to whatever is passed in \"fcnet_hiddens\" instead of being of the size of the action space. \r\n\r\n*Ray version and other system information (Python version, TensorFlow version, OS):*\r\n- ray==0.9.0.dev0 \r\n- python 3.6.10\r\n- mac OS\r\n\r\n### Reproduction (REQUIRED)\r\n\r\n```python\r\nimport ray\r\nfrom ray import tune\r\n\r\nray.init()\r\n\r\nconfig = {\r\n    \"env\": \"CartPole-v1\",\r\n    \"num_workers\": 1,\r\n    \"train_batch_size\": 128,\r\n    \"learning_starts\": 128,\r\n    \"model\": {\"fcnet_hiddens\": [32]},\r\n    \"dueling\": False ,\r\n    \"framework\": \"torch\"\r\n}\r\n\r\ntune.run(\"DQN\", name=\"MWE\", config=config, stop={\"training_iteration\": 100})\r\n```\r\n\r\n- [x] I have verified my script runs in a clean environment and reproduces the issue.\r\n- [x] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).\r\n\n", "before_files": [{"content": "import numpy as np\n\nfrom ray.rllib.models.torch.torch_modelv2 import TorchModelV2\nfrom ray.rllib.utils.framework import try_import_torch\n\ntorch, nn = try_import_torch()\n\n\nclass DQNTorchModel(TorchModelV2, nn.Module):\n    \"\"\"Extension of standard TorchModelV2 to provide dueling-Q functionality.\n    \"\"\"\n\n    def __init__(\n            self,\n            obs_space,\n            action_space,\n            num_outputs,\n            model_config,\n            name,\n            *,\n            dueling=False,\n            q_hiddens=(256, ),\n            dueling_activation=\"relu\",\n            use_noisy=False,\n            sigma0=0.5,\n            # TODO(sven): Move `add_layer_norm` into ModelCatalog as\n            #  generic option, then error if we use ParameterNoise as\n            #  Exploration type and do not have any LayerNorm layers in\n            #  the net.\n            add_layer_norm=False):\n        \"\"\"Initialize variables of this model.\n\n        Extra model kwargs:\n            dueling (bool): Whether to build the advantage(A)/value(V) heads\n                for DDQN. If True, Q-values are calculated as:\n                Q = (A - mean[A]) + V. If False, raw NN output is interpreted\n                as Q-values.\n            q_hiddens (List[int]): List of layer-sizes after(!) the\n                Advantages(A)/Value(V)-split. Hence, each of the A- and V-\n                branches will have this structure of Dense layers. To define\n                the NN before this A/V-split, use - as always -\n                config[\"model\"][\"fcnet_hiddens\"].\n            dueling_activation (str): The activation to use for all dueling\n                layers (A- and V-branch). One of \"relu\", \"tanh\", \"linear\".\n            use_noisy (bool): use noisy nets\n            sigma0 (float): initial value of noisy nets\n            add_layer_norm (bool): Enable layer norm (for param noise).\n        \"\"\"\n        nn.Module.__init__(self)\n        super(DQNTorchModel, self).__init__(obs_space, action_space,\n                                            num_outputs, model_config, name)\n\n        self.dueling = dueling\n        ins = num_outputs\n\n        # Dueling case: Build the shared (advantages and value) fc-network.\n        advantage_module = nn.Sequential()\n        value_module = None\n        if self.dueling:\n            value_module = nn.Sequential()\n            for i, n in enumerate(q_hiddens):\n                advantage_module.add_module(\"dueling_A_{}\".format(i),\n                                            nn.Linear(ins, n))\n                value_module.add_module(\"dueling_V_{}\".format(i),\n                                        nn.Linear(ins, n))\n                # Add activations if necessary.\n                if dueling_activation == \"relu\":\n                    advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n                                                nn.ReLU())\n                    value_module.add_module(\"dueling_V_act_{}\".format(i),\n                                            nn.ReLU())\n                elif dueling_activation == \"tanh\":\n                    advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n                                                nn.Tanh())\n                    value_module.add_module(\"dueling_V_act_{}\".format(i),\n                                            nn.Tanh())\n\n                # Add LayerNorm after each Dense.\n                if add_layer_norm:\n                    advantage_module.add_module(\"LayerNorm_A_{}\".format(i),\n                                                nn.LayerNorm(n))\n                    value_module.add_module(\"LayerNorm_V_{}\".format(i),\n                                            nn.LayerNorm(n))\n                ins = n\n            # Actual Advantages layer (nodes=num-actions) and\n            # value layer (nodes=1).\n            advantage_module.add_module(\"A\", nn.Linear(ins, action_space.n))\n            value_module.add_module(\"V\", nn.Linear(ins, 1))\n        # Non-dueling:\n        # Q-value layer (use main module's outputs as Q-values).\n        else:\n            pass\n\n        self.advantage_module = advantage_module\n        self.value_module = value_module\n\n    def get_advantages_or_q_values(self, model_out):\n        \"\"\"Returns distributional values for Q(s, a) given a state embedding.\n\n        Override this in your custom model to customize the Q output head.\n\n        Arguments:\n            model_out (Tensor): embedding from the model layers\n\n        Returns:\n            (action_scores, logits, dist) if num_atoms == 1, otherwise\n            (action_scores, z, support_logits_per_action, logits, dist)\n        \"\"\"\n\n        return self.advantage_module(model_out)\n\n    def get_state_value(self, model_out):\n        \"\"\"Returns the state value prediction for the given state embedding.\"\"\"\n\n        return self.value_module(model_out)\n\n    def _noisy_layer(self, action_in, out_size, sigma0, non_linear=True):\n        \"\"\"\n        a common dense layer: y = w^{T}x + b\n        a noisy layer: y = (w + \\\\epsilon_w*\\\\sigma_w)^{T}x +\n            (b+\\\\epsilon_b*\\\\sigma_b)\n        where \\epsilon are random variables sampled from factorized normal\n        distributions and \\\\sigma are trainable variables which are expected to\n        vanish along the training procedure\n        \"\"\"\n        in_size = int(action_in.shape[1])\n\n        epsilon_in = torch.normal(\n            mean=torch.zeros([in_size]), std=torch.ones([in_size]))\n        epsilon_out = torch.normal(\n            mean=torch.zeros([out_size]), std=torch.ones([out_size]))\n        epsilon_in = self._f_epsilon(epsilon_in)\n        epsilon_out = self._f_epsilon(epsilon_out)\n        epsilon_w = torch.matmul(\n            torch.unsqueeze(epsilon_in, -1),\n            other=torch.unsqueeze(epsilon_out, 0))\n        epsilon_b = epsilon_out\n\n        sigma_w = torch.Tensor(\n            data=np.random.uniform(\n                low=-1.0 / np.sqrt(float(in_size)),\n                high=1.0 / np.sqrt(float(in_size)),\n                size=[in_size, out_size]),\n            dtype=torch.float32,\n            requires_grad=True)\n        # TF noise generation can be unreliable on GPU\n        # If generating the noise on the CPU,\n        # lowering sigma0 to 0.1 may be helpful\n        sigma_b = torch.Tensor(\n            data=np.full(\n                shape=[out_size], fill_value=sigma0 / np.sqrt(float(in_size))),\n            requires_grad=True)\n        w = torch.Tensor(\n            data=np.full(\n                shape=[in_size, out_size],\n                fill_value=6 / np.sqrt(float(in_size) + float(out_size))),\n            requires_grad=True)\n        b = torch.Tensor(data=np.zeros([out_size]), requires_grad=True)\n        action_activation = torch.matmul(action_in, w + sigma_w * epsilon_w) \\\n            + b + sigma_b * epsilon_b\n\n        if not non_linear:\n            return action_activation\n        return nn.functional.relu(action_activation)\n\n    def _f_epsilon(self, x):\n        return torch.sign(x) * torch.pow(torch.abs(x), 0.5)\n", "path": "rllib/agents/dqn/dqn_torch_model.py"}, {"content": "\"\"\"Example of handling variable length and/or parametric action spaces.\n\nThis is a toy example of the action-embedding based approach for handling large\ndiscrete action spaces (potentially infinite in size), similar to this:\n\n    https://neuro.cs.ut.ee/the-use-of-embeddings-in-openai-five/\n\nThis currently works with RLlib's policy gradient style algorithms\n(e.g., PG, PPO, IMPALA, A2C) and also DQN.\n\nNote that since the model outputs now include \"-inf\" tf.float32.min\nvalues, not all algorithm options are supported at the moment. For example,\nalgorithms might crash if they don't properly ignore the -inf action scores.\nWorking configurations are given below.\n\"\"\"\n\nimport argparse\n\nimport ray\nfrom ray import tune\nfrom ray.rllib.examples.env.parametric_actions_cartpole import \\\n    ParametricActionsCartPole\nfrom ray.rllib.examples.models.parametric_actions_model import \\\n    ParametricActionsModel, TorchParametricActionsModel\nfrom ray.rllib.models import ModelCatalog\nfrom ray.rllib.utils.test_utils import check_learning_achieved\nfrom ray.tune.registry import register_env\n\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--run\", type=str, default=\"PPO\")\nparser.add_argument(\"--torch\", action=\"store_true\")\nparser.add_argument(\"--as-test\", action=\"store_true\")\nparser.add_argument(\"--stop-iters\", type=int, default=200)\nparser.add_argument(\"--stop-reward\", type=float, default=150.0)\nparser.add_argument(\"--stop-timesteps\", type=int, default=100000)\n\nif __name__ == \"__main__\":\n    args = parser.parse_args()\n    ray.init()\n\n    register_env(\"pa_cartpole\", lambda _: ParametricActionsCartPole(10))\n    ModelCatalog.register_custom_model(\n        \"pa_model\", TorchParametricActionsModel\n        if args.torch else ParametricActionsModel)\n\n    if args.run == \"DQN\":\n        cfg = {\n            # TODO(ekl) we need to set these to prevent the masked values\n            # from being further processed in DistributionalQModel, which\n            # would mess up the masking. It is possible to support these if we\n            # defined a a custom DistributionalQModel that is aware of masking.\n            \"hiddens\": [],\n            \"dueling\": False,\n        }\n    else:\n        cfg = {}\n\n    config = dict({\n        \"env\": \"pa_cartpole\",\n        \"model\": {\n            \"custom_model\": \"pa_model\",\n        },\n        \"num_workers\": 0,\n        \"framework\": \"torch\" if args.torch else \"tf\",\n    }, **cfg)\n\n    stop = {\n        \"training_iteration\": args.stop_iters,\n        \"timesteps_total\": args.stop_timesteps,\n        \"episode_reward_mean\": args.stop_reward,\n    }\n\n    results = tune.run(args.run, stop=stop, config=config, verbose=1)\n\n    if args.as_test:\n        check_learning_achieved(results, args.stop_reward)\n\n    ray.shutdown()\n", "path": "rllib/examples/parametric_actions_cartpole.py"}], "after_files": [{"content": "import numpy as np\n\nfrom ray.rllib.models.torch.torch_modelv2 import TorchModelV2\nfrom ray.rllib.utils.framework import try_import_torch\n\ntorch, nn = try_import_torch()\n\n\nclass DQNTorchModel(TorchModelV2, nn.Module):\n    \"\"\"Extension of standard TorchModelV2 to provide dueling-Q functionality.\n    \"\"\"\n\n    def __init__(\n            self,\n            obs_space,\n            action_space,\n            num_outputs,\n            model_config,\n            name,\n            *,\n            dueling=False,\n            q_hiddens=(256, ),\n            dueling_activation=\"relu\",\n            use_noisy=False,\n            sigma0=0.5,\n            # TODO(sven): Move `add_layer_norm` into ModelCatalog as\n            #  generic option, then error if we use ParameterNoise as\n            #  Exploration type and do not have any LayerNorm layers in\n            #  the net.\n            add_layer_norm=False):\n        \"\"\"Initialize variables of this model.\n\n        Extra model kwargs:\n            dueling (bool): Whether to build the advantage(A)/value(V) heads\n                for DDQN. If True, Q-values are calculated as:\n                Q = (A - mean[A]) + V. If False, raw NN output is interpreted\n                as Q-values.\n            q_hiddens (List[int]): List of layer-sizes after(!) the\n                Advantages(A)/Value(V)-split. Hence, each of the A- and V-\n                branches will have this structure of Dense layers. To define\n                the NN before this A/V-split, use - as always -\n                config[\"model\"][\"fcnet_hiddens\"].\n            dueling_activation (str): The activation to use for all dueling\n                layers (A- and V-branch). One of \"relu\", \"tanh\", \"linear\".\n            use_noisy (bool): use noisy nets\n            sigma0 (float): initial value of noisy nets\n            add_layer_norm (bool): Enable layer norm (for param noise).\n        \"\"\"\n        nn.Module.__init__(self)\n        super(DQNTorchModel, self).__init__(obs_space, action_space,\n                                            num_outputs, model_config, name)\n\n        self.dueling = dueling\n        ins = num_outputs\n\n        advantage_module = nn.Sequential()\n        value_module = nn.Sequential()\n\n        # Dueling case: Build the shared (advantages and value) fc-network.\n        for i, n in enumerate(q_hiddens):\n            advantage_module.add_module(\"dueling_A_{}\".format(i),\n                                        nn.Linear(ins, n))\n            value_module.add_module(\"dueling_V_{}\".format(i),\n                                    nn.Linear(ins, n))\n            # Add activations if necessary.\n            if dueling_activation == \"relu\":\n                advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n                                            nn.ReLU())\n                value_module.add_module(\"dueling_V_act_{}\".format(i),\n                                        nn.ReLU())\n            elif dueling_activation == \"tanh\":\n                advantage_module.add_module(\"dueling_A_act_{}\".format(i),\n                                            nn.Tanh())\n                value_module.add_module(\"dueling_V_act_{}\".format(i),\n                                        nn.Tanh())\n\n            # Add LayerNorm after each Dense.\n            if add_layer_norm:\n                advantage_module.add_module(\"LayerNorm_A_{}\".format(i),\n                                            nn.LayerNorm(n))\n                value_module.add_module(\"LayerNorm_V_{}\".format(i),\n                                        nn.LayerNorm(n))\n            ins = n\n\n        # Actual Advantages layer (nodes=num-actions).\n        if q_hiddens:\n            advantage_module.add_module(\"A\", nn.Linear(ins, action_space.n))\n\n        self.advantage_module = advantage_module\n\n        # Value layer (nodes=1).\n        if self.dueling:\n            value_module.add_module(\"V\", nn.Linear(ins, 1))\n            self.value_module = value_module\n\n    def get_advantages_or_q_values(self, model_out):\n        \"\"\"Returns distributional values for Q(s, a) given a state embedding.\n\n        Override this in your custom model to customize the Q output head.\n\n        Arguments:\n            model_out (Tensor): embedding from the model layers\n\n        Returns:\n            (action_scores, logits, dist) if num_atoms == 1, otherwise\n            (action_scores, z, support_logits_per_action, logits, dist)\n        \"\"\"\n\n        return self.advantage_module(model_out)\n\n    def get_state_value(self, model_out):\n        \"\"\"Returns the state value prediction for the given state embedding.\"\"\"\n\n        return self.value_module(model_out)\n\n    def _noisy_layer(self, action_in, out_size, sigma0, non_linear=True):\n        \"\"\"\n        a common dense layer: y = w^{T}x + b\n        a noisy layer: y = (w + \\\\epsilon_w*\\\\sigma_w)^{T}x +\n            (b+\\\\epsilon_b*\\\\sigma_b)\n        where \\epsilon are random variables sampled from factorized normal\n        distributions and \\\\sigma are trainable variables which are expected to\n        vanish along the training procedure\n        \"\"\"\n        in_size = int(action_in.shape[1])\n\n        epsilon_in = torch.normal(\n            mean=torch.zeros([in_size]), std=torch.ones([in_size]))\n        epsilon_out = torch.normal(\n            mean=torch.zeros([out_size]), std=torch.ones([out_size]))\n        epsilon_in = self._f_epsilon(epsilon_in)\n        epsilon_out = self._f_epsilon(epsilon_out)\n        epsilon_w = torch.matmul(\n            torch.unsqueeze(epsilon_in, -1),\n            other=torch.unsqueeze(epsilon_out, 0))\n        epsilon_b = epsilon_out\n\n        sigma_w = torch.Tensor(\n            data=np.random.uniform(\n                low=-1.0 / np.sqrt(float(in_size)),\n                high=1.0 / np.sqrt(float(in_size)),\n                size=[in_size, out_size]),\n            dtype=torch.float32,\n            requires_grad=True)\n        # TF noise generation can be unreliable on GPU\n        # If generating the noise on the CPU,\n        # lowering sigma0 to 0.1 may be helpful\n        sigma_b = torch.Tensor(\n            data=np.full(\n                shape=[out_size], fill_value=sigma0 / np.sqrt(float(in_size))),\n            requires_grad=True)\n        w = torch.Tensor(\n            data=np.full(\n                shape=[in_size, out_size],\n                fill_value=6 / np.sqrt(float(in_size) + float(out_size))),\n            requires_grad=True)\n        b = torch.Tensor(data=np.zeros([out_size]), requires_grad=True)\n        action_activation = torch.matmul(action_in, w + sigma_w * epsilon_w) \\\n            + b + sigma_b * epsilon_b\n\n        if not non_linear:\n            return action_activation\n        return nn.functional.relu(action_activation)\n\n    def _f_epsilon(self, x):\n        return torch.sign(x) * torch.pow(torch.abs(x), 0.5)\n", "path": "rllib/agents/dqn/dqn_torch_model.py"}, {"content": "\"\"\"Example of handling variable length and/or parametric action spaces.\n\nThis is a toy example of the action-embedding based approach for handling large\ndiscrete action spaces (potentially infinite in size), similar to this:\n\n    https://neuro.cs.ut.ee/the-use-of-embeddings-in-openai-five/\n\nThis currently works with RLlib's policy gradient style algorithms\n(e.g., PG, PPO, IMPALA, A2C) and also DQN.\n\nNote that since the model outputs now include \"-inf\" tf.float32.min\nvalues, not all algorithm options are supported at the moment. For example,\nalgorithms might crash if they don't properly ignore the -inf action scores.\nWorking configurations are given below.\n\"\"\"\n\nimport argparse\n\nimport ray\nfrom ray import tune\nfrom ray.rllib.examples.env.parametric_actions_cartpole import \\\n    ParametricActionsCartPole\nfrom ray.rllib.examples.models.parametric_actions_model import \\\n    ParametricActionsModel, TorchParametricActionsModel\nfrom ray.rllib.models import ModelCatalog\nfrom ray.rllib.utils.test_utils import check_learning_achieved\nfrom ray.tune.registry import register_env\n\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--run\", type=str, default=\"PPO\")\nparser.add_argument(\"--torch\", action=\"store_true\")\nparser.add_argument(\"--as-test\", action=\"store_true\")\nparser.add_argument(\"--stop-iters\", type=int, default=200)\nparser.add_argument(\"--stop-reward\", type=float, default=150.0)\nparser.add_argument(\"--stop-timesteps\", type=int, default=100000)\n\nif __name__ == \"__main__\":\n    args = parser.parse_args()\n    ray.init()\n\n    register_env(\"pa_cartpole\", lambda _: ParametricActionsCartPole(10))\n    ModelCatalog.register_custom_model(\n        \"pa_model\", TorchParametricActionsModel\n        if args.torch else ParametricActionsModel)\n\n    if args.run == \"DQN\":\n        cfg = {\n            # TODO(ekl) we need to set these to prevent the masked values\n            # from being further processed in DistributionalQModel, which\n            # would mess up the masking. It is possible to support these if we\n            # defined a custom DistributionalQModel that is aware of masking.\n            \"hiddens\": [],\n            \"dueling\": False,\n        }\n    else:\n        cfg = {}\n\n    config = dict({\n        \"env\": \"pa_cartpole\",\n        \"model\": {\n            \"custom_model\": \"pa_model\",\n        },\n        \"num_workers\": 0,\n        \"framework\": \"torch\" if args.torch else \"tf\",\n    }, **cfg)\n\n    stop = {\n        \"training_iteration\": args.stop_iters,\n        \"timesteps_total\": args.stop_timesteps,\n        \"episode_reward_mean\": args.stop_reward,\n    }\n\n    results = tune.run(args.run, stop=stop, config=config, verbose=1)\n\n    if args.as_test:\n        check_learning_achieved(results, args.stop_reward)\n\n    ray.shutdown()\n", "path": "rllib/examples/parametric_actions_cartpole.py"}]}