We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[rllib] Off Policy Estimation breaks with GPU on PyTorch (MARWIL) (Offline API)
### What is the problem?

When I use MARWIL with PyTorch and num_gpus: 1, I get an error when computing metrics. This happens because in off policy estimation it uses the torch tensors on gpu instead of numpy arrays. Particularly, I use "input_evaluation": ["is", "wis"] and the error goes away when "input_evaluation": ["simulation"]

```
Traceback (most recent call last):
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\tune\trial_runner.py", line 497, in _process_trial
    result = self.trial_executor.fetch_result(trial)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\tune\ray_trial_executor.py", line 434, in fetch_result
    result = ray.get(trial_future[0], DEFAULT_GET_TIMEOUT)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\worker.py", line 1553, in get
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(AttributeError): ray::MARWIL.train() (pid=9136, ip=10.0.0.18)
  File "python\ray\_raylet.pyx", line 474, in ray._raylet.execute_task
  File "python\ray\_raylet.pyx", line 427, in ray._raylet.execute_task.function_executor
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\function_manager.py", line 567, in actor_method_executor
    raise e
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\function_manager.py", line 559, in actor_method_executor
    method_returns = method(actor, *args, **kwargs)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\rllib\agents\trainer.py", line 522, in train
    raise e
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\rllib\agents\trainer.py", line 508, in train
    result = Trainable.train(self)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\tune\trainable.py", line 337, in train
    result = self.step()
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\rllib\agents\trainer_template.py", line 110, in step
    res = next(self.train_exec_impl)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\util\iter.py", line 758, in __next__
    return next(self.built_iterator)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\util\iter.py", line 793, in apply_foreach
    result = fn(item)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\rllib\execution\metric_ops.py", line 87, in __call__
    res = summarize_episodes(episodes, orig_episodes)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\ray\rllib\evaluation\metrics.py", line 173, in summarize_episodes
    metrics[k] = np.mean(v_list)
  File "<__array_function__ internals>", line 6, in mean
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\numpy\core\fromnumeric.py", line 3335, in mean
    out=out, **kwargs)
  File "C:\Users\Julius\Anaconda3\envs\ray\lib\site-packages\numpy\core\_methods.py", line 161, in _mean
    ret = ret.dtype.type(ret / rcount)
AttributeError: 'torch.dtype' object has no attribute 'type'
``` 

### Reproduction (REQUIRED)
Please provide a script that can be run to reproduce the issue. The script should have **no external library dependencies** (i.e., use fake or mock data / environments):

If we cannot run your script, we cannot fix your issue.

- [ ] I have verified my script runs in a clean environment and reproduces the issue.
- [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `rllib/offline/off_policy_estimator.py`
Content:
```
1 from collections import namedtuple
2 import logging
3 
4 from ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch
5 from ray.rllib.policy import Policy
6 from ray.rllib.utils.annotations import DeveloperAPI
7 from ray.rllib.offline.io_context import IOContext
8 from ray.rllib.utils.typing import TensorType, SampleBatchType
9 from typing import List
10 
11 logger = logging.getLogger(__name__)
12 
13 OffPolicyEstimate = namedtuple("OffPolicyEstimate",
14                                ["estimator_name", "metrics"])
15 
16 
17 @DeveloperAPI
18 class OffPolicyEstimator:
19     """Interface for an off policy reward estimator."""
20 
21     @DeveloperAPI
22     def __init__(self, policy: Policy, gamma: float):
23         """Creates an off-policy estimator.
24 
25         Arguments:
26             policy (Policy): Policy to evaluate.
27             gamma (float): Discount of the MDP.
28         """
29         self.policy = policy
30         self.gamma = gamma
31         self.new_estimates = []
32 
33     @classmethod
34     def create(cls, ioctx: IOContext) -> "OffPolicyEstimator":
35         """Create an off-policy estimator from a IOContext."""
36         gamma = ioctx.worker.policy_config["gamma"]
37         # Grab a reference to the current model
38         keys = list(ioctx.worker.policy_map.keys())
39         if len(keys) > 1:
40             raise NotImplementedError(
41                 "Off-policy estimation is not implemented for multi-agent. "
42                 "You can set `input_evaluation: []` to resolve this.")
43         policy = ioctx.worker.get_policy(keys[0])
44         return cls(policy, gamma)
45 
46     @DeveloperAPI
47     def estimate(self, batch: SampleBatchType):
48         """Returns an estimate for the given batch of experiences.
49 
50         The batch will only contain data from one episode, but it may only be
51         a fragment of an episode.
52         """
53         raise NotImplementedError
54 
55     @DeveloperAPI
56     def action_prob(self, batch: SampleBatchType) -> TensorType:
57         """Returns the probs for the batch actions for the current policy."""
58 
59         num_state_inputs = 0
60         for k in batch.keys():
61             if k.startswith("state_in_"):
62                 num_state_inputs += 1
63         state_keys = ["state_in_{}".format(i) for i in range(num_state_inputs)]
64         log_likelihoods = self.policy.compute_log_likelihoods(
65             actions=batch[SampleBatch.ACTIONS],
66             obs_batch=batch[SampleBatch.CUR_OBS],
67             state_batches=[batch[k] for k in state_keys],
68             prev_action_batch=batch.data.get(SampleBatch.PREV_ACTIONS),
69             prev_reward_batch=batch.data.get(SampleBatch.PREV_REWARDS))
70         return log_likelihoods
71 
72     @DeveloperAPI
73     def process(self, batch: SampleBatchType):
74         self.new_estimates.append(self.estimate(batch))
75 
76     @DeveloperAPI
77     def check_can_estimate_for(self, batch: SampleBatchType):
78         """Returns whether we can support OPE for this batch."""
79 
80         if isinstance(batch, MultiAgentBatch):
81             raise ValueError(
82                 "IS-estimation is not implemented for multi-agent batches. "
83                 "You can set `input_evaluation: []` to resolve this.")
84 
85         if "action_prob" not in batch:
86             raise ValueError(
87                 "Off-policy estimation is not possible unless the inputs "
88                 "include action probabilities (i.e., the policy is stochastic "
89                 "and emits the 'action_prob' key). For DQN this means using "
90                 "`exploration_config: {type: 'SoftQ'}`. You can also set "
91                 "`input_evaluation: []` to disable estimation.")
92 
93     @DeveloperAPI
94     def get_metrics(self) -> List[OffPolicyEstimate]:
95         """Return a list of new episode metric estimates since the last call.
96 
97         Returns:
98             list of OffPolicyEstimate objects.
99         """
100         out = self.new_estimates
101         self.new_estimates = []
102         return out
103
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/rllib/offline/off_policy_estimator.py b/rllib/offline/off_policy_estimator.py
--- a/rllib/offline/off_policy_estimator.py
+++ b/rllib/offline/off_policy_estimator.py
@@ -1,10 +1,13 @@
 from collections import namedtuple
 import logging
 
+import numpy as np
+
 from ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch
 from ray.rllib.policy import Policy
 from ray.rllib.utils.annotations import DeveloperAPI
 from ray.rllib.offline.io_context import IOContext
+from ray.rllib.utils.numpy import convert_to_numpy
 from ray.rllib.utils.typing import TensorType, SampleBatchType
 from typing import List
 
@@ -53,7 +56,7 @@
         raise NotImplementedError
 
     @DeveloperAPI
-    def action_prob(self, batch: SampleBatchType) -> TensorType:
+    def action_prob(self, batch: SampleBatchType) -> np.ndarray:
         """Returns the probs for the batch actions for the current policy."""
 
         num_state_inputs = 0
@@ -61,13 +64,13 @@
             if k.startswith("state_in_"):
                 num_state_inputs += 1
         state_keys = ["state_in_{}".format(i) for i in range(num_state_inputs)]
-        log_likelihoods = self.policy.compute_log_likelihoods(
+        log_likelihoods: TensorType = self.policy.compute_log_likelihoods(
             actions=batch[SampleBatch.ACTIONS],
             obs_batch=batch[SampleBatch.CUR_OBS],
             state_batches=[batch[k] for k in state_keys],
             prev_action_batch=batch.data.get(SampleBatch.PREV_ACTIONS),
             prev_reward_batch=batch.data.get(SampleBatch.PREV_REWARDS))
-        return log_likelihoods
+        return convert_to_numpy(log_likelihoods)
 
     @DeveloperAPI
     def process(self, batch: SampleBatchType):

{"golden_diff": "diff --git a/rllib/offline/off_policy_estimator.py b/rllib/offline/off_policy_estimator.py\n--- a/rllib/offline/off_policy_estimator.py\n+++ b/rllib/offline/off_policy_estimator.py\n@@ -1,10 +1,13 @@\n from collections import namedtuple\n import logging\n \n+import numpy as np\n+\n from ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch\n from ray.rllib.policy import Policy\n from ray.rllib.utils.annotations import DeveloperAPI\n from ray.rllib.offline.io_context import IOContext\n+from ray.rllib.utils.numpy import convert_to_numpy\n from ray.rllib.utils.typing import TensorType, SampleBatchType\n from typing import List\n \n@@ -53,7 +56,7 @@\n         raise NotImplementedError\n \n     @DeveloperAPI\n-    def action_prob(self, batch: SampleBatchType) -> TensorType:\n+    def action_prob(self, batch: SampleBatchType) -> np.ndarray:\n         \"\"\"Returns the probs for the batch actions for the current policy.\"\"\"\n \n         num_state_inputs = 0\n@@ -61,13 +64,13 @@\n             if k.startswith(\"state_in_\"):\n                 num_state_inputs += 1\n         state_keys = [\"state_in_{}\".format(i) for i in range(num_state_inputs)]\n-        log_likelihoods = self.policy.compute_log_likelihoods(\n+        log_likelihoods: TensorType = self.policy.compute_log_likelihoods(\n             actions=batch[SampleBatch.ACTIONS],\n             obs_batch=batch[SampleBatch.CUR_OBS],\n             state_batches=[batch[k] for k in state_keys],\n             prev_action_batch=batch.data.get(SampleBatch.PREV_ACTIONS),\n             prev_reward_batch=batch.data.get(SampleBatch.PREV_REWARDS))\n-        return log_likelihoods\n+        return convert_to_numpy(log_likelihoods)\n \n     @DeveloperAPI\n     def process(self, batch: SampleBatchType):\n", "issue": "[rllib] Off Policy Estimation breaks with GPU on PyTorch (MARWIL) (Offline API)\n### What is the problem?\r\n\r\nWhen I use MARWIL with PyTorch and num_gpus: 1, I get an error when computing metrics. This happens because in off policy estimation it uses the torch tensors on gpu instead of numpy arrays. Particularly, I use \"input_evaluation\": [\"is\", \"wis\"] and the error goes away when \"input_evaluation\": [\"simulation\"]\r\n\r\n```\r\nTraceback (most recent call last):\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\tune\\trial_runner.py\", line 497, in _process_trial\r\n    result = self.trial_executor.fetch_result(trial)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\tune\\ray_trial_executor.py\", line 434, in fetch_result\r\n    result = ray.get(trial_future[0], DEFAULT_GET_TIMEOUT)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\worker.py\", line 1553, in get\r\n    raise value.as_instanceof_cause()\r\nray.exceptions.RayTaskError(AttributeError): ray::MARWIL.train() (pid=9136, ip=10.0.0.18)\r\n  File \"python\\ray\\_raylet.pyx\", line 474, in ray._raylet.execute_task\r\n  File \"python\\ray\\_raylet.pyx\", line 427, in ray._raylet.execute_task.function_executor\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\function_manager.py\", line 567, in actor_method_executor\r\n    raise e\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\function_manager.py\", line 559, in actor_method_executor\r\n    method_returns = method(actor, *args, **kwargs)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\rllib\\agents\\trainer.py\", line 522, in train\r\n    raise e\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\rllib\\agents\\trainer.py\", line 508, in train\r\n    result = Trainable.train(self)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\tune\\trainable.py\", line 337, in train\r\n    result = self.step()\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\rllib\\agents\\trainer_template.py\", line 110, in step\r\n    res = next(self.train_exec_impl)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\util\\iter.py\", line 758, in __next__\r\n    return next(self.built_iterator)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\util\\iter.py\", line 793, in apply_foreach\r\n    result = fn(item)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\rllib\\execution\\metric_ops.py\", line 87, in __call__\r\n    res = summarize_episodes(episodes, orig_episodes)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\ray\\rllib\\evaluation\\metrics.py\", line 173, in summarize_episodes\r\n    metrics[k] = np.mean(v_list)\r\n  File \"<__array_function__ internals>\", line 6, in mean\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\numpy\\core\\fromnumeric.py\", line 3335, in mean\r\n    out=out, **kwargs)\r\n  File \"C:\\Users\\Julius\\Anaconda3\\envs\\ray\\lib\\site-packages\\numpy\\core\\_methods.py\", line 161, in _mean\r\n    ret = ret.dtype.type(ret / rcount)\r\nAttributeError: 'torch.dtype' object has no attribute 'type'\r\n``` \r\n\r\n### Reproduction (REQUIRED)\r\nPlease provide a script that can be run to reproduce the issue. The script should have **no external library dependencies** (i.e., use fake or mock data / environments):\r\n\r\nIf we cannot run your script, we cannot fix your issue.\r\n\r\n- [ ] I have verified my script runs in a clean environment and reproduces the issue.\r\n- [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).\r\n\n", "before_files": [{"content": "from collections import namedtuple\nimport logging\n\nfrom ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch\nfrom ray.rllib.policy import Policy\nfrom ray.rllib.utils.annotations import DeveloperAPI\nfrom ray.rllib.offline.io_context import IOContext\nfrom ray.rllib.utils.typing import TensorType, SampleBatchType\nfrom typing import List\n\nlogger = logging.getLogger(__name__)\n\nOffPolicyEstimate = namedtuple(\"OffPolicyEstimate\",\n                               [\"estimator_name\", \"metrics\"])\n\n\n@DeveloperAPI\nclass OffPolicyEstimator:\n    \"\"\"Interface for an off policy reward estimator.\"\"\"\n\n    @DeveloperAPI\n    def __init__(self, policy: Policy, gamma: float):\n        \"\"\"Creates an off-policy estimator.\n\n        Arguments:\n            policy (Policy): Policy to evaluate.\n            gamma (float): Discount of the MDP.\n        \"\"\"\n        self.policy = policy\n        self.gamma = gamma\n        self.new_estimates = []\n\n    @classmethod\n    def create(cls, ioctx: IOContext) -> \"OffPolicyEstimator\":\n        \"\"\"Create an off-policy estimator from a IOContext.\"\"\"\n        gamma = ioctx.worker.policy_config[\"gamma\"]\n        # Grab a reference to the current model\n        keys = list(ioctx.worker.policy_map.keys())\n        if len(keys) > 1:\n            raise NotImplementedError(\n                \"Off-policy estimation is not implemented for multi-agent. \"\n                \"You can set `input_evaluation: []` to resolve this.\")\n        policy = ioctx.worker.get_policy(keys[0])\n        return cls(policy, gamma)\n\n    @DeveloperAPI\n    def estimate(self, batch: SampleBatchType):\n        \"\"\"Returns an estimate for the given batch of experiences.\n\n        The batch will only contain data from one episode, but it may only be\n        a fragment of an episode.\n        \"\"\"\n        raise NotImplementedError\n\n    @DeveloperAPI\n    def action_prob(self, batch: SampleBatchType) -> TensorType:\n        \"\"\"Returns the probs for the batch actions for the current policy.\"\"\"\n\n        num_state_inputs = 0\n        for k in batch.keys():\n            if k.startswith(\"state_in_\"):\n                num_state_inputs += 1\n        state_keys = [\"state_in_{}\".format(i) for i in range(num_state_inputs)]\n        log_likelihoods = self.policy.compute_log_likelihoods(\n            actions=batch[SampleBatch.ACTIONS],\n            obs_batch=batch[SampleBatch.CUR_OBS],\n            state_batches=[batch[k] for k in state_keys],\n            prev_action_batch=batch.data.get(SampleBatch.PREV_ACTIONS),\n            prev_reward_batch=batch.data.get(SampleBatch.PREV_REWARDS))\n        return log_likelihoods\n\n    @DeveloperAPI\n    def process(self, batch: SampleBatchType):\n        self.new_estimates.append(self.estimate(batch))\n\n    @DeveloperAPI\n    def check_can_estimate_for(self, batch: SampleBatchType):\n        \"\"\"Returns whether we can support OPE for this batch.\"\"\"\n\n        if isinstance(batch, MultiAgentBatch):\n            raise ValueError(\n                \"IS-estimation is not implemented for multi-agent batches. \"\n                \"You can set `input_evaluation: []` to resolve this.\")\n\n        if \"action_prob\" not in batch:\n            raise ValueError(\n                \"Off-policy estimation is not possible unless the inputs \"\n                \"include action probabilities (i.e., the policy is stochastic \"\n                \"and emits the 'action_prob' key). For DQN this means using \"\n                \"`exploration_config: {type: 'SoftQ'}`. You can also set \"\n                \"`input_evaluation: []` to disable estimation.\")\n\n    @DeveloperAPI\n    def get_metrics(self) -> List[OffPolicyEstimate]:\n        \"\"\"Return a list of new episode metric estimates since the last call.\n\n        Returns:\n            list of OffPolicyEstimate objects.\n        \"\"\"\n        out = self.new_estimates\n        self.new_estimates = []\n        return out\n", "path": "rllib/offline/off_policy_estimator.py"}], "after_files": [{"content": "from collections import namedtuple\nimport logging\n\nimport numpy as np\n\nfrom ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch\nfrom ray.rllib.policy import Policy\nfrom ray.rllib.utils.annotations import DeveloperAPI\nfrom ray.rllib.offline.io_context import IOContext\nfrom ray.rllib.utils.numpy import convert_to_numpy\nfrom ray.rllib.utils.typing import TensorType, SampleBatchType\nfrom typing import List\n\nlogger = logging.getLogger(__name__)\n\nOffPolicyEstimate = namedtuple(\"OffPolicyEstimate\",\n                               [\"estimator_name\", \"metrics\"])\n\n\n@DeveloperAPI\nclass OffPolicyEstimator:\n    \"\"\"Interface for an off policy reward estimator.\"\"\"\n\n    @DeveloperAPI\n    def __init__(self, policy: Policy, gamma: float):\n        \"\"\"Creates an off-policy estimator.\n\n        Arguments:\n            policy (Policy): Policy to evaluate.\n            gamma (float): Discount of the MDP.\n        \"\"\"\n        self.policy = policy\n        self.gamma = gamma\n        self.new_estimates = []\n\n    @classmethod\n    def create(cls, ioctx: IOContext) -> \"OffPolicyEstimator\":\n        \"\"\"Create an off-policy estimator from a IOContext.\"\"\"\n        gamma = ioctx.worker.policy_config[\"gamma\"]\n        # Grab a reference to the current model\n        keys = list(ioctx.worker.policy_map.keys())\n        if len(keys) > 1:\n            raise NotImplementedError(\n                \"Off-policy estimation is not implemented for multi-agent. \"\n                \"You can set `input_evaluation: []` to resolve this.\")\n        policy = ioctx.worker.get_policy(keys[0])\n        return cls(policy, gamma)\n\n    @DeveloperAPI\n    def estimate(self, batch: SampleBatchType):\n        \"\"\"Returns an estimate for the given batch of experiences.\n\n        The batch will only contain data from one episode, but it may only be\n        a fragment of an episode.\n        \"\"\"\n        raise NotImplementedError\n\n    @DeveloperAPI\n    def action_prob(self, batch: SampleBatchType) -> np.ndarray:\n        \"\"\"Returns the probs for the batch actions for the current policy.\"\"\"\n\n        num_state_inputs = 0\n        for k in batch.keys():\n            if k.startswith(\"state_in_\"):\n                num_state_inputs += 1\n        state_keys = [\"state_in_{}\".format(i) for i in range(num_state_inputs)]\n        log_likelihoods: TensorType = self.policy.compute_log_likelihoods(\n            actions=batch[SampleBatch.ACTIONS],\n            obs_batch=batch[SampleBatch.CUR_OBS],\n            state_batches=[batch[k] for k in state_keys],\n            prev_action_batch=batch.data.get(SampleBatch.PREV_ACTIONS),\n            prev_reward_batch=batch.data.get(SampleBatch.PREV_REWARDS))\n        return convert_to_numpy(log_likelihoods)\n\n    @DeveloperAPI\n    def process(self, batch: SampleBatchType):\n        self.new_estimates.append(self.estimate(batch))\n\n    @DeveloperAPI\n    def check_can_estimate_for(self, batch: SampleBatchType):\n        \"\"\"Returns whether we can support OPE for this batch.\"\"\"\n\n        if isinstance(batch, MultiAgentBatch):\n            raise ValueError(\n                \"IS-estimation is not implemented for multi-agent batches. \"\n                \"You can set `input_evaluation: []` to resolve this.\")\n\n        if \"action_prob\" not in batch:\n            raise ValueError(\n                \"Off-policy estimation is not possible unless the inputs \"\n                \"include action probabilities (i.e., the policy is stochastic \"\n                \"and emits the 'action_prob' key). For DQN this means using \"\n                \"`exploration_config: {type: 'SoftQ'}`. You can also set \"\n                \"`input_evaluation: []` to disable estimation.\")\n\n    @DeveloperAPI\n    def get_metrics(self) -> List[OffPolicyEstimate]:\n        \"\"\"Return a list of new episode metric estimates since the last call.\n\n        Returns:\n            list of OffPolicyEstimate objects.\n        \"\"\"\n        out = self.new_estimates\n        self.new_estimates = []\n        return out\n", "path": "rllib/offline/off_policy_estimator.py"}]}