We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
API: change CLs to be scalar
# Description

right now it's returning (1,) vectors


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/pyhf/cli/infer.py`
Content:
```
1 """The inference CLI group."""
2 import logging
3 
4 import click
5 import json
6 
7 from ..utils import EqDelimStringParamType
8 from ..infer import hypotest
9 from ..workspace import Workspace
10 from .. import get_backend, set_backend, optimize
11 
12 log = logging.getLogger(__name__)
13 
14 
15 @click.group(name='infer')
16 def cli():
17     """Infererence CLI group."""
18 
19 
20 @cli.command()
21 @click.argument('workspace', default='-')
22 @click.option(
23     '--output-file',
24     help='The location of the output json file. If not specified, prints to screen.',
25     default=None,
26 )
27 @click.option('--measurement', default=None)
28 @click.option('-p', '--patch', multiple=True)
29 @click.option('--testpoi', default=1.0)
30 @click.option('--teststat', type=click.Choice(['q', 'qtilde']), default='qtilde')
31 @click.option(
32     '--backend',
33     type=click.Choice(['numpy', 'pytorch', 'tensorflow', 'jax', 'np', 'torch', 'tf']),
34     help='The tensor backend used for the calculation.',
35     default='numpy',
36 )
37 @click.option('--optimizer')
38 @click.option('--optconf', type=EqDelimStringParamType(), multiple=True)
39 def cls(
40     workspace,
41     output_file,
42     measurement,
43     patch,
44     testpoi,
45     teststat,
46     backend,
47     optimizer,
48     optconf,
49 ):
50     """
51     Compute CLs value(s) for a given pyhf workspace.
52 
53     Example:
54 
55     .. code-block:: shell
56 
57         $ curl -sL https://git.io/JJYDE | pyhf cls
58 
59         \b
60         {
61             "CLs_exp": [
62                 0.07807427911686156,
63                 0.17472571775474618,
64                 0.35998495263681285,
65                 0.6343568235898907,
66                 0.8809947004472013
67             ],
68             "CLs_obs": 0.3599845631401915
69         }
70     """
71     with click.open_file(workspace, 'r') as specstream:
72         spec = json.load(specstream)
73 
74     ws = Workspace(spec)
75 
76     is_qtilde = teststat == 'qtilde'
77 
78     patches = [json.loads(click.open_file(pfile, 'r').read()) for pfile in patch]
79     model = ws.model(
80         measurement_name=measurement,
81         patches=patches,
82         modifier_settings={
83             'normsys': {'interpcode': 'code4'},
84             'histosys': {'interpcode': 'code4p'},
85         },
86     )
87 
88     # set the backend if not NumPy
89     if backend in ['pytorch', 'torch']:
90         set_backend("pytorch", precision="64b")
91     elif backend in ['tensorflow', 'tf']:
92         set_backend("tensorflow", precision="64b")
93     elif backend in ['jax']:
94         set_backend("jax")
95     tensorlib, _ = get_backend()
96 
97     optconf = {k: v for item in optconf for k, v in item.items()}
98 
99     # set the new optimizer
100     if optimizer:
101         new_optimizer = getattr(optimize, optimizer) or getattr(
102             optimize, f'{optimizer}_optimizer'
103         )
104         set_backend(tensorlib, new_optimizer(**optconf))
105 
106     result = hypotest(
107         testpoi, ws.data(model), model, qtilde=is_qtilde, return_expected_set=True
108     )
109     result = {
110         'CLs_obs': tensorlib.tolist(result[0])[0],
111         'CLs_exp': tensorlib.tolist(tensorlib.reshape(result[-1], [-1])),
112     }
113 
114     if output_file is None:
115         click.echo(json.dumps(result, indent=4, sort_keys=True))
116     else:
117         with open(output_file, 'w+') as out_file:
118             json.dump(result, out_file, indent=4, sort_keys=True)
119         log.debug("Written to {0:s}".format(output_file))
120
```

Path: `src/pyhf/infer/__init__.py`
Content:
```
1 """Inference for Statistical Models."""
2 
3 from .test_statistics import qmu
4 from .. import get_backend
5 from .calculators import AsymptoticCalculator
6 
7 
8 def hypotest(
9     poi_test, data, pdf, init_pars=None, par_bounds=None, qtilde=False, **kwargs
10 ):
11     r"""
12     Compute :math:`p`-values and test statistics for a single value of the parameter of interest.
13 
14     Example:
15         >>> import pyhf
16         >>> pyhf.set_backend("numpy")
17         >>> model = pyhf.simplemodels.hepdata_like(
18         ...     signal_data=[12.0, 11.0], bkg_data=[50.0, 52.0], bkg_uncerts=[3.0, 7.0]
19         ... )
20         >>> observations = [51, 48]
21         >>> data = pyhf.tensorlib.astensor(observations + model.config.auxdata)
22         >>> test_poi = 1.0
23         >>> CLs_obs, CLs_exp_band = pyhf.infer.hypotest(
24         ...     test_poi, data, model, qtilde=True, return_expected_set=True
25         ... )
26         >>> print(CLs_obs)
27         [0.05251554]
28         >>> print(CLs_exp_band)
29         [[0.00260641]
30          [0.01382066]
31          [0.06445521]
32          [0.23526104]
33          [0.57304182]]
34 
35     Args:
36         poi_test (Number or Tensor): The value of the parameter of interest (POI)
37         data (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\sqrt{q_{\mu,A}}`
38         pdf (~pyhf.pdf.Model): The HistFactory statistical model
39         init_pars (Array or Tensor): The initial parameter values to be used for minimization
40         par_bounds (Array or Tensor): The parameter value bounds to be used for minimization
41         qtilde (Bool): When ``True`` perform the calculation using the alternative test statistic, :math:`\tilde{q}`, as defined in Equation (62) of :xref:`arXiv:1007.1727`
42 
43     Keyword Args:
44         return_tail_probs (bool): Bool for returning :math:`\textrm{CL}_{s+b}` and :math:`\textrm{CL}_{b}`
45         return_expected (bool): Bool for returning :math:`\textrm{CL}_{\textrm{exp}}`
46         return_expected_set (bool): Bool for returning the :math:`(-2,-1,0,1,2)\sigma` :math:`\textrm{CL}_{\textrm{exp}}` --- the "Brazil band"
47 
48     Returns:
49         Tuple of Floats and lists of Floats:
50 
51             - :math:`\textrm{CL}_{s}`: The :math:`p`-value compared to the given threshold :math:`\alpha`, typically taken to be :math:`0.05`, defined in :xref:`arXiv:1007.1727` as
52 
53             .. math::
54 
55                 \textrm{CL}_{s} = \frac{\textrm{CL}_{s+b}}{\textrm{CL}_{b}} = \frac{p_{s+b}}{1-p_{b}}
56 
57             to protect against excluding signal models in which there is little sensitivity. In the case that :math:`\textrm{CL}_{s} \leq \alpha` the given signal model is excluded.
58 
59             - :math:`\left[\textrm{CL}_{s+b}, \textrm{CL}_{b}\right]`: The signal + background :math:`p`-value and 1 minus the background only :math:`p`-value as defined in Equations (75) and (76) of :xref:`arXiv:1007.1727`
60 
61             .. math::
62 
63                 \textrm{CL}_{s+b} = p_{s+b} = \int\limits_{q_{\textrm{obs}}}^{\infty} f\left(q\,\middle|s+b\right)\,dq = 1 - \Phi\left(\frac{q_{\textrm{obs}} + 1/\sigma_{s+b}^{2}}{2/\sigma_{s+b}}\right)
64 
65             .. math::
66 
67                 \textrm{CL}_{b} = 1- p_{b} = 1 - \int\limits_{-\infty}^{q_{\textrm{obs}}} f\left(q\,\middle|b\right)\,dq = 1 - \Phi\left(\frac{q_{\textrm{obs}} - 1/\sigma_{b}^{2}}{2/\sigma_{b}}\right)
68 
69             with Equations (73) and (74) for the mean
70 
71             .. math::
72 
73                 E\left[q\right] = \frac{1 - 2\mu}{\sigma^{2}}
74 
75             and variance
76 
77             .. math::
78 
79                 V\left[q\right] = \frac{4}{\sigma^{2}}
80 
81             of the test statistic :math:`q` under the background only and and signal + background hypotheses. Only returned when ``return_tail_probs`` is ``True``.
82 
83             - :math:`\textrm{CL}_{s,\textrm{exp}}`: The expected :math:`\textrm{CL}_{s}` value corresponding to the test statistic under the background only hypothesis :math:`\left(\mu=0\right)`. Only returned when ``return_expected`` is ``True``.
84 
85             - :math:`\textrm{CL}_{s,\textrm{exp}}` band: The set of expected :math:`\textrm{CL}_{s}` values corresponding to the median significance of variations of the signal strength from the background only hypothesis :math:`\left(\mu=0\right)` at :math:`(-2,-1,0,1,2)\sigma`. That is, the :math:`p`-values that satisfy Equation (89) of :xref:`arXiv:1007.1727`
86 
87             .. math::
88 
89                 \textrm{band}_{N\sigma} = \mu' + \sigma\,\Phi^{-1}\left(1-\alpha\right) \pm N\sigma
90 
91             for :math:`\mu'=0` and :math:`N \in \left\{-2, -1, 0, 1, 2\right\}`. These values define the boundaries of an uncertainty band sometimes referred to as the "Brazil band". Only returned when ``return_expected_set`` is ``True``.
92 
93     """
94     init_pars = init_pars or pdf.config.suggested_init()
95     par_bounds = par_bounds or pdf.config.suggested_bounds()
96     tensorlib, _ = get_backend()
97 
98     calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde)
99     teststat = calc.teststatistic(poi_test)
100     sig_plus_bkg_distribution, b_only_distribution = calc.distributions(poi_test)
101 
102     CLsb = sig_plus_bkg_distribution.pvalue(teststat)
103     CLb = b_only_distribution.pvalue(teststat)
104     CLs = CLsb / CLb
105     CLsb, CLb, CLs = (
106         tensorlib.reshape(CLsb, (1,)),
107         tensorlib.reshape(CLb, (1,)),
108         tensorlib.reshape(CLs, (1,)),
109     )
110 
111     _returns = [CLs]
112     if kwargs.get('return_tail_probs'):
113         _returns.append([CLsb, CLb])
114     if kwargs.get('return_expected_set'):
115         CLs_exp = []
116         for n_sigma in [2, 1, 0, -1, -2]:
117 
118             expected_bonly_teststat = b_only_distribution.expected_value(n_sigma)
119 
120             CLs = sig_plus_bkg_distribution.pvalue(
121                 expected_bonly_teststat
122             ) / b_only_distribution.pvalue(expected_bonly_teststat)
123             CLs_exp.append(tensorlib.reshape(CLs, (1,)))
124         CLs_exp = tensorlib.astensor(CLs_exp)
125         if kwargs.get('return_expected'):
126             _returns.append(CLs_exp[2])
127         _returns.append(CLs_exp)
128     elif kwargs.get('return_expected'):
129         n_sigma = 0
130         expected_bonly_teststat = b_only_distribution.expected_value(n_sigma)
131 
132         CLs = sig_plus_bkg_distribution.pvalue(
133             expected_bonly_teststat
134         ) / b_only_distribution.pvalue(expected_bonly_teststat)
135         _returns.append(tensorlib.reshape(CLs, (1,)))
136     # Enforce a consistent return type of the observed CLs
137     return tuple(_returns) if len(_returns) > 1 else _returns[0]
138 
139 
140 __all__ = ['qmu', 'hypotest']
141
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/pyhf/cli/infer.py b/src/pyhf/cli/infer.py
--- a/src/pyhf/cli/infer.py
+++ b/src/pyhf/cli/infer.py
@@ -107,8 +107,8 @@
         testpoi, ws.data(model), model, qtilde=is_qtilde, return_expected_set=True
     )
     result = {
-        'CLs_obs': tensorlib.tolist(result[0])[0],
-        'CLs_exp': tensorlib.tolist(tensorlib.reshape(result[-1], [-1])),
+        'CLs_obs': tensorlib.tolist(result[0]),
+        'CLs_exp': [tensorlib.tolist(tensor) for tensor in result[-1]],
     }
 
     if output_file is None:
diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py
--- a/src/pyhf/infer/__init__.py
+++ b/src/pyhf/infer/__init__.py
@@ -23,14 +23,10 @@
         >>> CLs_obs, CLs_exp_band = pyhf.infer.hypotest(
         ...     test_poi, data, model, qtilde=True, return_expected_set=True
         ... )
-        >>> print(CLs_obs)
-        [0.05251554]
-        >>> print(CLs_exp_band)
-        [[0.00260641]
-         [0.01382066]
-         [0.06445521]
-         [0.23526104]
-         [0.57304182]]
+        >>> CLs_obs
+        array(0.05251554)
+        >>> CLs_exp_band
+        [array(0.00260641), array(0.01382066), array(0.06445521), array(0.23526104), array(0.57304182)]
 
     Args:
         poi_test (Number or Tensor): The value of the parameter of interest (POI)
@@ -102,10 +98,11 @@
     CLsb = sig_plus_bkg_distribution.pvalue(teststat)
     CLb = b_only_distribution.pvalue(teststat)
     CLs = CLsb / CLb
+    # Ensure that all CL values are 0-d tensors
     CLsb, CLb, CLs = (
-        tensorlib.reshape(CLsb, (1,)),
-        tensorlib.reshape(CLb, (1,)),
-        tensorlib.reshape(CLs, (1,)),
+        tensorlib.astensor(CLsb),
+        tensorlib.astensor(CLb),
+        tensorlib.astensor(CLs),
     )
 
     _returns = [CLs]
@@ -120,8 +117,7 @@
             CLs = sig_plus_bkg_distribution.pvalue(
                 expected_bonly_teststat
             ) / b_only_distribution.pvalue(expected_bonly_teststat)
-            CLs_exp.append(tensorlib.reshape(CLs, (1,)))
-        CLs_exp = tensorlib.astensor(CLs_exp)
+            CLs_exp.append(tensorlib.astensor(CLs))
         if kwargs.get('return_expected'):
             _returns.append(CLs_exp[2])
         _returns.append(CLs_exp)
@@ -132,7 +128,7 @@
         CLs = sig_plus_bkg_distribution.pvalue(
             expected_bonly_teststat
         ) / b_only_distribution.pvalue(expected_bonly_teststat)
-        _returns.append(tensorlib.reshape(CLs, (1,)))
+        _returns.append(tensorlib.astensor(CLs))
     # Enforce a consistent return type of the observed CLs
     return tuple(_returns) if len(_returns) > 1 else _returns[0]

{"golden_diff": "diff --git a/src/pyhf/cli/infer.py b/src/pyhf/cli/infer.py\n--- a/src/pyhf/cli/infer.py\n+++ b/src/pyhf/cli/infer.py\n@@ -107,8 +107,8 @@\n         testpoi, ws.data(model), model, qtilde=is_qtilde, return_expected_set=True\n     )\n     result = {\n-        'CLs_obs': tensorlib.tolist(result[0])[0],\n-        'CLs_exp': tensorlib.tolist(tensorlib.reshape(result[-1], [-1])),\n+        'CLs_obs': tensorlib.tolist(result[0]),\n+        'CLs_exp': [tensorlib.tolist(tensor) for tensor in result[-1]],\n     }\n \n     if output_file is None:\ndiff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py\n--- a/src/pyhf/infer/__init__.py\n+++ b/src/pyhf/infer/__init__.py\n@@ -23,14 +23,10 @@\n         >>> CLs_obs, CLs_exp_band = pyhf.infer.hypotest(\n         ...     test_poi, data, model, qtilde=True, return_expected_set=True\n         ... )\n-        >>> print(CLs_obs)\n-        [0.05251554]\n-        >>> print(CLs_exp_band)\n-        [[0.00260641]\n-         [0.01382066]\n-         [0.06445521]\n-         [0.23526104]\n-         [0.57304182]]\n+        >>> CLs_obs\n+        array(0.05251554)\n+        >>> CLs_exp_band\n+        [array(0.00260641), array(0.01382066), array(0.06445521), array(0.23526104), array(0.57304182)]\n \n     Args:\n         poi_test (Number or Tensor): The value of the parameter of interest (POI)\n@@ -102,10 +98,11 @@\n     CLsb = sig_plus_bkg_distribution.pvalue(teststat)\n     CLb = b_only_distribution.pvalue(teststat)\n     CLs = CLsb / CLb\n+    # Ensure that all CL values are 0-d tensors\n     CLsb, CLb, CLs = (\n-        tensorlib.reshape(CLsb, (1,)),\n-        tensorlib.reshape(CLb, (1,)),\n-        tensorlib.reshape(CLs, (1,)),\n+        tensorlib.astensor(CLsb),\n+        tensorlib.astensor(CLb),\n+        tensorlib.astensor(CLs),\n     )\n \n     _returns = [CLs]\n@@ -120,8 +117,7 @@\n             CLs = sig_plus_bkg_distribution.pvalue(\n                 expected_bonly_teststat\n             ) / b_only_distribution.pvalue(expected_bonly_teststat)\n-            CLs_exp.append(tensorlib.reshape(CLs, (1,)))\n-        CLs_exp = tensorlib.astensor(CLs_exp)\n+            CLs_exp.append(tensorlib.astensor(CLs))\n         if kwargs.get('return_expected'):\n             _returns.append(CLs_exp[2])\n         _returns.append(CLs_exp)\n@@ -132,7 +128,7 @@\n         CLs = sig_plus_bkg_distribution.pvalue(\n             expected_bonly_teststat\n         ) / b_only_distribution.pvalue(expected_bonly_teststat)\n-        _returns.append(tensorlib.reshape(CLs, (1,)))\n+        _returns.append(tensorlib.astensor(CLs))\n     # Enforce a consistent return type of the observed CLs\n     return tuple(_returns) if len(_returns) > 1 else _returns[0]\n", "issue": "API: change CLs to be scalar\n# Description\r\n\r\nright now it's returning (1,) vectors\r\n\n", "before_files": [{"content": "\"\"\"The inference CLI group.\"\"\"\nimport logging\n\nimport click\nimport json\n\nfrom ..utils import EqDelimStringParamType\nfrom ..infer import hypotest\nfrom ..workspace import Workspace\nfrom .. import get_backend, set_backend, optimize\n\nlog = logging.getLogger(__name__)\n\n\n@click.group(name='infer')\ndef cli():\n    \"\"\"Infererence CLI group.\"\"\"\n\n\n@cli.command()\n@click.argument('workspace', default='-')\n@click.option(\n    '--output-file',\n    help='The location of the output json file. If not specified, prints to screen.',\n    default=None,\n)\n@click.option('--measurement', default=None)\n@click.option('-p', '--patch', multiple=True)\n@click.option('--testpoi', default=1.0)\n@click.option('--teststat', type=click.Choice(['q', 'qtilde']), default='qtilde')\n@click.option(\n    '--backend',\n    type=click.Choice(['numpy', 'pytorch', 'tensorflow', 'jax', 'np', 'torch', 'tf']),\n    help='The tensor backend used for the calculation.',\n    default='numpy',\n)\n@click.option('--optimizer')\n@click.option('--optconf', type=EqDelimStringParamType(), multiple=True)\ndef cls(\n    workspace,\n    output_file,\n    measurement,\n    patch,\n    testpoi,\n    teststat,\n    backend,\n    optimizer,\n    optconf,\n):\n    \"\"\"\n    Compute CLs value(s) for a given pyhf workspace.\n\n    Example:\n\n    .. code-block:: shell\n\n        $ curl -sL https://git.io/JJYDE | pyhf cls\n\n        \\b\n        {\n            \"CLs_exp\": [\n                0.07807427911686156,\n                0.17472571775474618,\n                0.35998495263681285,\n                0.6343568235898907,\n                0.8809947004472013\n            ],\n            \"CLs_obs\": 0.3599845631401915\n        }\n    \"\"\"\n    with click.open_file(workspace, 'r') as specstream:\n        spec = json.load(specstream)\n\n    ws = Workspace(spec)\n\n    is_qtilde = teststat == 'qtilde'\n\n    patches = [json.loads(click.open_file(pfile, 'r').read()) for pfile in patch]\n    model = ws.model(\n        measurement_name=measurement,\n        patches=patches,\n        modifier_settings={\n            'normsys': {'interpcode': 'code4'},\n            'histosys': {'interpcode': 'code4p'},\n        },\n    )\n\n    # set the backend if not NumPy\n    if backend in ['pytorch', 'torch']:\n        set_backend(\"pytorch\", precision=\"64b\")\n    elif backend in ['tensorflow', 'tf']:\n        set_backend(\"tensorflow\", precision=\"64b\")\n    elif backend in ['jax']:\n        set_backend(\"jax\")\n    tensorlib, _ = get_backend()\n\n    optconf = {k: v for item in optconf for k, v in item.items()}\n\n    # set the new optimizer\n    if optimizer:\n        new_optimizer = getattr(optimize, optimizer) or getattr(\n            optimize, f'{optimizer}_optimizer'\n        )\n        set_backend(tensorlib, new_optimizer(**optconf))\n\n    result = hypotest(\n        testpoi, ws.data(model), model, qtilde=is_qtilde, return_expected_set=True\n    )\n    result = {\n        'CLs_obs': tensorlib.tolist(result[0])[0],\n        'CLs_exp': tensorlib.tolist(tensorlib.reshape(result[-1], [-1])),\n    }\n\n    if output_file is None:\n        click.echo(json.dumps(result, indent=4, sort_keys=True))\n    else:\n        with open(output_file, 'w+') as out_file:\n            json.dump(result, out_file, indent=4, sort_keys=True)\n        log.debug(\"Written to {0:s}\".format(output_file))\n", "path": "src/pyhf/cli/infer.py"}, {"content": "\"\"\"Inference for Statistical Models.\"\"\"\n\nfrom .test_statistics import qmu\nfrom .. import get_backend\nfrom .calculators import AsymptoticCalculator\n\n\ndef hypotest(\n    poi_test, data, pdf, init_pars=None, par_bounds=None, qtilde=False, **kwargs\n):\n    r\"\"\"\n    Compute :math:`p`-values and test statistics for a single value of the parameter of interest.\n\n    Example:\n        >>> import pyhf\n        >>> pyhf.set_backend(\"numpy\")\n        >>> model = pyhf.simplemodels.hepdata_like(\n        ...     signal_data=[12.0, 11.0], bkg_data=[50.0, 52.0], bkg_uncerts=[3.0, 7.0]\n        ... )\n        >>> observations = [51, 48]\n        >>> data = pyhf.tensorlib.astensor(observations + model.config.auxdata)\n        >>> test_poi = 1.0\n        >>> CLs_obs, CLs_exp_band = pyhf.infer.hypotest(\n        ...     test_poi, data, model, qtilde=True, return_expected_set=True\n        ... )\n        >>> print(CLs_obs)\n        [0.05251554]\n        >>> print(CLs_exp_band)\n        [[0.00260641]\n         [0.01382066]\n         [0.06445521]\n         [0.23526104]\n         [0.57304182]]\n\n    Args:\n        poi_test (Number or Tensor): The value of the parameter of interest (POI)\n        data (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\\sqrt{q_{\\mu,A}}`\n        pdf (~pyhf.pdf.Model): The HistFactory statistical model\n        init_pars (Array or Tensor): The initial parameter values to be used for minimization\n        par_bounds (Array or Tensor): The parameter value bounds to be used for minimization\n        qtilde (Bool): When ``True`` perform the calculation using the alternative test statistic, :math:`\\tilde{q}`, as defined in Equation (62) of :xref:`arXiv:1007.1727`\n\n    Keyword Args:\n        return_tail_probs (bool): Bool for returning :math:`\\textrm{CL}_{s+b}` and :math:`\\textrm{CL}_{b}`\n        return_expected (bool): Bool for returning :math:`\\textrm{CL}_{\\textrm{exp}}`\n        return_expected_set (bool): Bool for returning the :math:`(-2,-1,0,1,2)\\sigma` :math:`\\textrm{CL}_{\\textrm{exp}}` --- the \"Brazil band\"\n\n    Returns:\n        Tuple of Floats and lists of Floats:\n\n            - :math:`\\textrm{CL}_{s}`: The :math:`p`-value compared to the given threshold :math:`\\alpha`, typically taken to be :math:`0.05`, defined in :xref:`arXiv:1007.1727` as\n\n            .. math::\n\n                \\textrm{CL}_{s} = \\frac{\\textrm{CL}_{s+b}}{\\textrm{CL}_{b}} = \\frac{p_{s+b}}{1-p_{b}}\n\n            to protect against excluding signal models in which there is little sensitivity. In the case that :math:`\\textrm{CL}_{s} \\leq \\alpha` the given signal model is excluded.\n\n            - :math:`\\left[\\textrm{CL}_{s+b}, \\textrm{CL}_{b}\\right]`: The signal + background :math:`p`-value and 1 minus the background only :math:`p`-value as defined in Equations (75) and (76) of :xref:`arXiv:1007.1727`\n\n            .. math::\n\n                \\textrm{CL}_{s+b} = p_{s+b} = \\int\\limits_{q_{\\textrm{obs}}}^{\\infty} f\\left(q\\,\\middle|s+b\\right)\\,dq = 1 - \\Phi\\left(\\frac{q_{\\textrm{obs}} + 1/\\sigma_{s+b}^{2}}{2/\\sigma_{s+b}}\\right)\n\n            .. math::\n\n                \\textrm{CL}_{b} = 1- p_{b} = 1 - \\int\\limits_{-\\infty}^{q_{\\textrm{obs}}} f\\left(q\\,\\middle|b\\right)\\,dq = 1 - \\Phi\\left(\\frac{q_{\\textrm{obs}} - 1/\\sigma_{b}^{2}}{2/\\sigma_{b}}\\right)\n\n            with Equations (73) and (74) for the mean\n\n            .. math::\n\n                E\\left[q\\right] = \\frac{1 - 2\\mu}{\\sigma^{2}}\n\n            and variance\n\n            .. math::\n\n                V\\left[q\\right] = \\frac{4}{\\sigma^{2}}\n\n            of the test statistic :math:`q` under the background only and and signal + background hypotheses. Only returned when ``return_tail_probs`` is ``True``.\n\n            - :math:`\\textrm{CL}_{s,\\textrm{exp}}`: The expected :math:`\\textrm{CL}_{s}` value corresponding to the test statistic under the background only hypothesis :math:`\\left(\\mu=0\\right)`. Only returned when ``return_expected`` is ``True``.\n\n            - :math:`\\textrm{CL}_{s,\\textrm{exp}}` band: The set of expected :math:`\\textrm{CL}_{s}` values corresponding to the median significance of variations of the signal strength from the background only hypothesis :math:`\\left(\\mu=0\\right)` at :math:`(-2,-1,0,1,2)\\sigma`. That is, the :math:`p`-values that satisfy Equation (89) of :xref:`arXiv:1007.1727`\n\n            .. math::\n\n                \\textrm{band}_{N\\sigma} = \\mu' + \\sigma\\,\\Phi^{-1}\\left(1-\\alpha\\right) \\pm N\\sigma\n\n            for :math:`\\mu'=0` and :math:`N \\in \\left\\{-2, -1, 0, 1, 2\\right\\}`. These values define the boundaries of an uncertainty band sometimes referred to as the \"Brazil band\". Only returned when ``return_expected_set`` is ``True``.\n\n    \"\"\"\n    init_pars = init_pars or pdf.config.suggested_init()\n    par_bounds = par_bounds or pdf.config.suggested_bounds()\n    tensorlib, _ = get_backend()\n\n    calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde)\n    teststat = calc.teststatistic(poi_test)\n    sig_plus_bkg_distribution, b_only_distribution = calc.distributions(poi_test)\n\n    CLsb = sig_plus_bkg_distribution.pvalue(teststat)\n    CLb = b_only_distribution.pvalue(teststat)\n    CLs = CLsb / CLb\n    CLsb, CLb, CLs = (\n        tensorlib.reshape(CLsb, (1,)),\n        tensorlib.reshape(CLb, (1,)),\n        tensorlib.reshape(CLs, (1,)),\n    )\n\n    _returns = [CLs]\n    if kwargs.get('return_tail_probs'):\n        _returns.append([CLsb, CLb])\n    if kwargs.get('return_expected_set'):\n        CLs_exp = []\n        for n_sigma in [2, 1, 0, -1, -2]:\n\n            expected_bonly_teststat = b_only_distribution.expected_value(n_sigma)\n\n            CLs = sig_plus_bkg_distribution.pvalue(\n                expected_bonly_teststat\n            ) / b_only_distribution.pvalue(expected_bonly_teststat)\n            CLs_exp.append(tensorlib.reshape(CLs, (1,)))\n        CLs_exp = tensorlib.astensor(CLs_exp)\n        if kwargs.get('return_expected'):\n            _returns.append(CLs_exp[2])\n        _returns.append(CLs_exp)\n    elif kwargs.get('return_expected'):\n        n_sigma = 0\n        expected_bonly_teststat = b_only_distribution.expected_value(n_sigma)\n\n        CLs = sig_plus_bkg_distribution.pvalue(\n            expected_bonly_teststat\n        ) / b_only_distribution.pvalue(expected_bonly_teststat)\n        _returns.append(tensorlib.reshape(CLs, (1,)))\n    # Enforce a consistent return type of the observed CLs\n    return tuple(_returns) if len(_returns) > 1 else _returns[0]\n\n\n__all__ = ['qmu', 'hypotest']\n", "path": "src/pyhf/infer/__init__.py"}], "after_files": [{"content": "\"\"\"The inference CLI group.\"\"\"\nimport logging\n\nimport click\nimport json\n\nfrom ..utils import EqDelimStringParamType\nfrom ..infer import hypotest\nfrom ..workspace import Workspace\nfrom .. import get_backend, set_backend, optimize\n\nlog = logging.getLogger(__name__)\n\n\n@click.group(name='infer')\ndef cli():\n    \"\"\"Infererence CLI group.\"\"\"\n\n\n@cli.command()\n@click.argument('workspace', default='-')\n@click.option(\n    '--output-file',\n    help='The location of the output json file. If not specified, prints to screen.',\n    default=None,\n)\n@click.option('--measurement', default=None)\n@click.option('-p', '--patch', multiple=True)\n@click.option('--testpoi', default=1.0)\n@click.option('--teststat', type=click.Choice(['q', 'qtilde']), default='qtilde')\n@click.option(\n    '--backend',\n    type=click.Choice(['numpy', 'pytorch', 'tensorflow', 'jax', 'np', 'torch', 'tf']),\n    help='The tensor backend used for the calculation.',\n    default='numpy',\n)\n@click.option('--optimizer')\n@click.option('--optconf', type=EqDelimStringParamType(), multiple=True)\ndef cls(\n    workspace,\n    output_file,\n    measurement,\n    patch,\n    testpoi,\n    teststat,\n    backend,\n    optimizer,\n    optconf,\n):\n    \"\"\"\n    Compute CLs value(s) for a given pyhf workspace.\n\n    Example:\n\n    .. code-block:: shell\n\n        $ curl -sL https://git.io/JJYDE | pyhf cls\n\n        \\b\n        {\n            \"CLs_exp\": [\n                0.07807427911686156,\n                0.17472571775474618,\n                0.35998495263681285,\n                0.6343568235898907,\n                0.8809947004472013\n            ],\n            \"CLs_obs\": 0.3599845631401915\n        }\n    \"\"\"\n    with click.open_file(workspace, 'r') as specstream:\n        spec = json.load(specstream)\n\n    ws = Workspace(spec)\n\n    is_qtilde = teststat == 'qtilde'\n\n    patches = [json.loads(click.open_file(pfile, 'r').read()) for pfile in patch]\n    model = ws.model(\n        measurement_name=measurement,\n        patches=patches,\n        modifier_settings={\n            'normsys': {'interpcode': 'code4'},\n            'histosys': {'interpcode': 'code4p'},\n        },\n    )\n\n    # set the backend if not NumPy\n    if backend in ['pytorch', 'torch']:\n        set_backend(\"pytorch\", precision=\"64b\")\n    elif backend in ['tensorflow', 'tf']:\n        set_backend(\"tensorflow\", precision=\"64b\")\n    elif backend in ['jax']:\n        set_backend(\"jax\")\n    tensorlib, _ = get_backend()\n\n    optconf = {k: v for item in optconf for k, v in item.items()}\n\n    # set the new optimizer\n    if optimizer:\n        new_optimizer = getattr(optimize, optimizer) or getattr(\n            optimize, f'{optimizer}_optimizer'\n        )\n        set_backend(tensorlib, new_optimizer(**optconf))\n\n    result = hypotest(\n        testpoi, ws.data(model), model, qtilde=is_qtilde, return_expected_set=True\n    )\n    result = {\n        'CLs_obs': tensorlib.tolist(result[0]),\n        'CLs_exp': [tensorlib.tolist(tensor) for tensor in result[-1]],\n    }\n\n    if output_file is None:\n        click.echo(json.dumps(result, indent=4, sort_keys=True))\n    else:\n        with open(output_file, 'w+') as out_file:\n            json.dump(result, out_file, indent=4, sort_keys=True)\n        log.debug(\"Written to {0:s}\".format(output_file))\n", "path": "src/pyhf/cli/infer.py"}, {"content": "\"\"\"Inference for Statistical Models.\"\"\"\n\nfrom .test_statistics import qmu\nfrom .. import get_backend\nfrom .calculators import AsymptoticCalculator\n\n\ndef hypotest(\n    poi_test, data, pdf, init_pars=None, par_bounds=None, qtilde=False, **kwargs\n):\n    r\"\"\"\n    Compute :math:`p`-values and test statistics for a single value of the parameter of interest.\n\n    Example:\n        >>> import pyhf\n        >>> pyhf.set_backend(\"numpy\")\n        >>> model = pyhf.simplemodels.hepdata_like(\n        ...     signal_data=[12.0, 11.0], bkg_data=[50.0, 52.0], bkg_uncerts=[3.0, 7.0]\n        ... )\n        >>> observations = [51, 48]\n        >>> data = pyhf.tensorlib.astensor(observations + model.config.auxdata)\n        >>> test_poi = 1.0\n        >>> CLs_obs, CLs_exp_band = pyhf.infer.hypotest(\n        ...     test_poi, data, model, qtilde=True, return_expected_set=True\n        ... )\n        >>> CLs_obs\n        array(0.05251554)\n        >>> CLs_exp_band\n        [array(0.00260641), array(0.01382066), array(0.06445521), array(0.23526104), array(0.57304182)]\n\n    Args:\n        poi_test (Number or Tensor): The value of the parameter of interest (POI)\n        data (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\\sqrt{q_{\\mu,A}}`\n        pdf (~pyhf.pdf.Model): The HistFactory statistical model\n        init_pars (Array or Tensor): The initial parameter values to be used for minimization\n        par_bounds (Array or Tensor): The parameter value bounds to be used for minimization\n        qtilde (Bool): When ``True`` perform the calculation using the alternative test statistic, :math:`\\tilde{q}`, as defined in Equation (62) of :xref:`arXiv:1007.1727`\n\n    Keyword Args:\n        return_tail_probs (bool): Bool for returning :math:`\\textrm{CL}_{s+b}` and :math:`\\textrm{CL}_{b}`\n        return_expected (bool): Bool for returning :math:`\\textrm{CL}_{\\textrm{exp}}`\n        return_expected_set (bool): Bool for returning the :math:`(-2,-1,0,1,2)\\sigma` :math:`\\textrm{CL}_{\\textrm{exp}}` --- the \"Brazil band\"\n\n    Returns:\n        Tuple of Floats and lists of Floats:\n\n            - :math:`\\textrm{CL}_{s}`: The :math:`p`-value compared to the given threshold :math:`\\alpha`, typically taken to be :math:`0.05`, defined in :xref:`arXiv:1007.1727` as\n\n            .. math::\n\n                \\textrm{CL}_{s} = \\frac{\\textrm{CL}_{s+b}}{\\textrm{CL}_{b}} = \\frac{p_{s+b}}{1-p_{b}}\n\n            to protect against excluding signal models in which there is little sensitivity. In the case that :math:`\\textrm{CL}_{s} \\leq \\alpha` the given signal model is excluded.\n\n            - :math:`\\left[\\textrm{CL}_{s+b}, \\textrm{CL}_{b}\\right]`: The signal + background :math:`p`-value and 1 minus the background only :math:`p`-value as defined in Equations (75) and (76) of :xref:`arXiv:1007.1727`\n\n            .. math::\n\n                \\textrm{CL}_{s+b} = p_{s+b} = \\int\\limits_{q_{\\textrm{obs}}}^{\\infty} f\\left(q\\,\\middle|s+b\\right)\\,dq = 1 - \\Phi\\left(\\frac{q_{\\textrm{obs}} + 1/\\sigma_{s+b}^{2}}{2/\\sigma_{s+b}}\\right)\n\n            .. math::\n\n                \\textrm{CL}_{b} = 1- p_{b} = 1 - \\int\\limits_{-\\infty}^{q_{\\textrm{obs}}} f\\left(q\\,\\middle|b\\right)\\,dq = 1 - \\Phi\\left(\\frac{q_{\\textrm{obs}} - 1/\\sigma_{b}^{2}}{2/\\sigma_{b}}\\right)\n\n            with Equations (73) and (74) for the mean\n\n            .. math::\n\n                E\\left[q\\right] = \\frac{1 - 2\\mu}{\\sigma^{2}}\n\n            and variance\n\n            .. math::\n\n                V\\left[q\\right] = \\frac{4}{\\sigma^{2}}\n\n            of the test statistic :math:`q` under the background only and and signal + background hypotheses. Only returned when ``return_tail_probs`` is ``True``.\n\n            - :math:`\\textrm{CL}_{s,\\textrm{exp}}`: The expected :math:`\\textrm{CL}_{s}` value corresponding to the test statistic under the background only hypothesis :math:`\\left(\\mu=0\\right)`. Only returned when ``return_expected`` is ``True``.\n\n            - :math:`\\textrm{CL}_{s,\\textrm{exp}}` band: The set of expected :math:`\\textrm{CL}_{s}` values corresponding to the median significance of variations of the signal strength from the background only hypothesis :math:`\\left(\\mu=0\\right)` at :math:`(-2,-1,0,1,2)\\sigma`. That is, the :math:`p`-values that satisfy Equation (89) of :xref:`arXiv:1007.1727`\n\n            .. math::\n\n                \\textrm{band}_{N\\sigma} = \\mu' + \\sigma\\,\\Phi^{-1}\\left(1-\\alpha\\right) \\pm N\\sigma\n\n            for :math:`\\mu'=0` and :math:`N \\in \\left\\{-2, -1, 0, 1, 2\\right\\}`. These values define the boundaries of an uncertainty band sometimes referred to as the \"Brazil band\". Only returned when ``return_expected_set`` is ``True``.\n\n    \"\"\"\n    init_pars = init_pars or pdf.config.suggested_init()\n    par_bounds = par_bounds or pdf.config.suggested_bounds()\n    tensorlib, _ = get_backend()\n\n    calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde)\n    teststat = calc.teststatistic(poi_test)\n    sig_plus_bkg_distribution, b_only_distribution = calc.distributions(poi_test)\n\n    CLsb = sig_plus_bkg_distribution.pvalue(teststat)\n    CLb = b_only_distribution.pvalue(teststat)\n    CLs = CLsb / CLb\n    # Ensure that all CL values are 0-d tensors\n    CLsb, CLb, CLs = (\n        tensorlib.astensor(CLsb),\n        tensorlib.astensor(CLb),\n        tensorlib.astensor(CLs),\n    )\n\n    _returns = [CLs]\n    if kwargs.get('return_tail_probs'):\n        _returns.append([CLsb, CLb])\n    if kwargs.get('return_expected_set'):\n        CLs_exp = []\n        for n_sigma in [2, 1, 0, -1, -2]:\n\n            expected_bonly_teststat = b_only_distribution.expected_value(n_sigma)\n\n            CLs = sig_plus_bkg_distribution.pvalue(\n                expected_bonly_teststat\n            ) / b_only_distribution.pvalue(expected_bonly_teststat)\n            CLs_exp.append(tensorlib.astensor(CLs))\n        if kwargs.get('return_expected'):\n            _returns.append(CLs_exp[2])\n        _returns.append(CLs_exp)\n    elif kwargs.get('return_expected'):\n        n_sigma = 0\n        expected_bonly_teststat = b_only_distribution.expected_value(n_sigma)\n\n        CLs = sig_plus_bkg_distribution.pvalue(\n            expected_bonly_teststat\n        ) / b_only_distribution.pvalue(expected_bonly_teststat)\n        _returns.append(tensorlib.astensor(CLs))\n    # Enforce a consistent return type of the observed CLs\n    return tuple(_returns) if len(_returns) > 1 else _returns[0]\n\n\n__all__ = ['qmu', 'hypotest']\n", "path": "src/pyhf/infer/__init__.py"}]}