We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
`eps` is doubly added to variance in inference of `L.BatchRenormalization`
- `runninng_var` learns variances with `eps`
- `train=False` mode uses `running_var + eps`

### Conditions
I tested with Chainer versions: `3.0.0`, `4.5.0`, `5.4.0`, `7.0.0a1`

### Code to reproduce
```python
import chainer
import numpy as np
np.random.seed(0)

brn = chainer.links.BatchRenormalization(3, eps=1.)
for _ in range(1000):
    x = np.random.randn(1000, 3).astype('f')
    brn(x)

x = np.random.randn(1000, 3).astype('f')

y = brn(x)
print(y.array.var(axis=0))

with chainer.using_config('train', False):
    y = brn(x)
print(y.array.var(axis=0))
```

### Error messages, stack traces, or logs
```
[0.51281106 0.49953052 0.48790243]
[0.3506052  0.33283928 0.31892547]
```

Here, the "normalized" variance is around 1/(1+eps) if train, whereas it's around 1/(1+2 eps) otherwise.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `chainer/functions/normalization/batch_renormalization.py`
Content:
```
1 import warnings
2 
3 import numpy
4 
5 from chainer import backend
6 from chainer.backends import cuda
7 from chainer import configuration
8 from chainer import function
9 from chainer.functions.normalization import batch_normalization
10 from chainer.utils import type_check
11 
12 
13 def _xhat(x, mean, std, expander):
14     x_mu = x - mean[expander]
15     x_mu /= std[expander]
16     return x_mu
17 
18 
19 class BatchRenormalizationFunction(function.Function):
20 
21     def __init__(self, eps=2e-5, mean=None, var=None, decay=0.9,
22                  rmax=1, dmax=0, update_statistics=True):
23         self._running_mean = mean
24         self._running_var = var
25         self.rmax = rmax
26         self.dmax = dmax
27         self.r = None
28         self.update_statistics = update_statistics
29 
30         self.eps = eps
31         self.decay = decay
32 
33     def _warn_accessing_property(self):
34         warnings.warn(
35             'The attributes of BatchRenormalizationFunction '
36             'are deprecated. '
37             'Consider setting update_statistics=True to '
38             'batch_renormalization to update running statistics.',
39             DeprecationWarning)
40 
41     @property
42     def running_mean(self):
43         self._warn_accessing_property()
44         return self._running_mean
45 
46     @property
47     def running_var(self):
48         self._warn_accessing_property()
49         return self._running_var
50 
51     def check_type_forward(self, in_types):
52         type_check.expect(in_types.size() == 3)
53         x_type, gamma_type, beta_type = in_types
54         M = type_check.eval(gamma_type.ndim)
55         type_check.expect(
56             x_type.dtype.kind == 'f',
57             x_type.ndim >= gamma_type.ndim + 1,
58             x_type.shape[1:1 + M] == gamma_type.shape,
59             # TODO(tkerola): Check shape
60             gamma_type.dtype.kind == 'f',
61             gamma_type.dtype == beta_type.dtype,
62             gamma_type.shape == beta_type.shape,
63         )
64 
65     def forward(self, inputs):
66         xp = backend.get_array_module(*inputs)
67         x, gamma, beta = inputs
68 
69         # Note: we must be in train mode.
70         assert configuration.config.train
71 
72         head_ndim = gamma.ndim + 1
73         expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)
74 
75         # NOTE(tommi): cuDNN is not used since it does not support
76         # batch renormalization
77         axis = (0,) + tuple(range(head_ndim, x.ndim))
78         mean = x.mean(axis=axis, dtype=gamma.dtype)
79         var = x.var(axis=axis, dtype=gamma.dtype) + self.eps
80         self.std = xp.sqrt(var, dtype=var.dtype)
81 
82         running_sigma = xp.sqrt(self._running_var + self.eps,
83                                 dtype=self._running_mean.dtype)
84         self.r = xp.clip(self.std / running_sigma,
85                          1.0 / self.rmax, self.rmax)
86         d = xp.clip(
87             (mean - self._running_mean) / running_sigma,
88             -self.dmax, self.dmax)
89 
90         gamma = gamma[expander]
91         beta = beta[expander]
92 
93         if xp is numpy:
94             self.x_hat = _xhat(x, mean, self.std, expander)
95             self.x_hat_renorm = self.x_hat * self.r[expander] + d[expander]
96             y = gamma * self.x_hat_renorm
97             y += beta
98             y = y.astype(dtype=x.dtype)
99         else:
100             self.x_hat, self.x_hat_renorm, y = cuda.elementwise(
101                 'T x, U mean, U std, U gamma, U beta, U r, U d',
102                 'U x_hat, U x_hat_renorm, T y',
103                 '''
104                 x_hat = (x - mean) / std;
105                 x_hat_renorm = x_hat * r + d;
106                 y = gamma * x_hat_renorm + beta;
107                 ''',
108                 'brn_fwd')(
109                     x, mean[expander], self.std[expander], gamma, beta,
110                     self.r[expander], d[expander])
111 
112         if self.update_statistics:
113             m = x.size // gamma[expander].size
114             self._running_mean *= self.decay
115             adjust = m / max(m - 1., 1.)  # unbiased estimation
116             temp_ar = xp.array(mean)
117             temp_ar *= (1 - self.decay)
118             self._running_mean += temp_ar
119             del temp_ar
120             self._running_var *= self.decay
121             temp_ar = xp.array(var)
122             temp_ar *= (1 - self.decay) * adjust
123             self._running_var += temp_ar
124             del temp_ar
125 
126         return y,
127 
128     def backward(self, inputs, grad_outputs):
129         x, gamma, _ = inputs
130         gy = grad_outputs[0]
131         head_ndim = gamma.ndim + 1
132         expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)
133         m = gamma.dtype.type(x.size // gamma.size)
134         axis = (0,) + tuple(range(head_ndim, x.ndim))
135         xp = backend.get_array_module(x)
136 
137         # Note: we must be in train mode.
138         assert configuration.config.train
139         # NOTE(tommi): cuDNN is not used since it does not support
140         # batch renormalization
141         gbeta = gy.sum(axis=axis, dtype=gamma.dtype)
142         ggamma = (gy * self.x_hat_renorm).sum(axis=axis)
143         gsigma_batch = (gy * self.x_hat).sum(axis=axis)
144         if xp is numpy:
145             scale = (self.r * gamma / self.std)[expander]
146             gx = scale * (gy - (self.x_hat * gsigma_batch[expander] +
147                                 gbeta[expander]) / m)
148             gx = gx.astype(dtype=x.dtype)
149         else:
150             inv_m = numpy.float32(1) / m
151             gx = cuda.elementwise(
152                 'T gy, U x_hat, U gamma, U std, U gsigma_batch, U gbeta, \
153                 U inv_m, U r',
154                 'T gx',
155                 'gx = (r * gamma / std) * (gy - (x_hat * gsigma_batch + gbeta) * \
156                 inv_m)',
157                 'brn_bwd')(
158                     gy, self.x_hat, gamma[expander],
159                     self.std[expander], gsigma_batch[expander],
160                     gbeta[expander], inv_m, self.r[expander])
161         return gx, ggamma, gbeta
162 
163 
164 def batch_renormalization(x, gamma, beta, rmax, dmax, eps=2e-5,
165                           running_mean=None, running_var=None, decay=0.9,
166                           update_statistics=False):
167     """Batch renormalization function.
168 
169     This is an extension of batch normalization, which ensures that the
170     training and inference models generate the same outputs that depend on
171     individual examples rather than the entire minibatch.
172 
173     .. note::
174 
175         This function does not perform in-place update to
176         ``running_mean`` and ``running_var`` by default, contrary to
177         :func:`~chainer.functions.batch_normalization`.
178         If the function is called, it will not be possible to access the
179         updated running mean and variance statistics, because they are members
180         of the function object, which cannot be accessed by the caller.
181         If it is desired to update the running statistics, call the function
182         with ``update_statistics=True`` option.
183 
184     .. note::
185 
186         For the consistency with Batch Normalization, this function
187         intentionally ignores some of the theoretical flaws in Algorithm 1 of
188         the Batch Renormalization paper:
189 
190         - ``F.batch_renormalization`` maintains the moving average of variances
191           :math:`\\sigma^2`, while the original paper maintains the moving
192           average of standard deviations :math:`\\sigma`.
193         - ``F.batch_renormalization`` applies Bessel's correction to update the
194           moving average of variances.
195 
196     See: `Batch Renormalization: Towards Reducing Minibatch Dependence in
197     Batch-Normalized Models <https://arxiv.org/abs/1702.03275>`_
198 
199     .. seealso:: :class:`~chainer.links.BatchRenormalization`
200 
201     """
202     if running_mean is None:
203         raise TypeError('running_mean is required')
204     if running_var is None:
205         raise TypeError('running_var is required')
206     return BatchRenormalizationFunction(
207         eps, running_mean, running_var, decay, rmax, dmax, update_statistics
208     )(x, gamma, beta)
209 
210 
211 def fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5):
212     warnings.warn(
213         'fixed_batch_renormalization is deprecated. '
214         'Use fixed_batch_normalization instead.',
215         DeprecationWarning)
216     with configuration.using_config('train', False):
217         return batch_normalization.fixed_batch_normalization(
218             x, gamma, beta, mean, var, eps
219         )
220
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/chainer/functions/normalization/batch_renormalization.py b/chainer/functions/normalization/batch_renormalization.py
--- a/chainer/functions/normalization/batch_renormalization.py
+++ b/chainer/functions/normalization/batch_renormalization.py
@@ -76,8 +76,8 @@
         # batch renormalization
         axis = (0,) + tuple(range(head_ndim, x.ndim))
         mean = x.mean(axis=axis, dtype=gamma.dtype)
-        var = x.var(axis=axis, dtype=gamma.dtype) + self.eps
-        self.std = xp.sqrt(var, dtype=var.dtype)
+        var = x.var(axis=axis, dtype=gamma.dtype)
+        self.std = xp.sqrt(var + self.eps, dtype=var.dtype)
 
         running_sigma = xp.sqrt(self._running_var + self.eps,
                                 dtype=self._running_mean.dtype)

{"golden_diff": "diff --git a/chainer/functions/normalization/batch_renormalization.py b/chainer/functions/normalization/batch_renormalization.py\n--- a/chainer/functions/normalization/batch_renormalization.py\n+++ b/chainer/functions/normalization/batch_renormalization.py\n@@ -76,8 +76,8 @@\n         # batch renormalization\n         axis = (0,) + tuple(range(head_ndim, x.ndim))\n         mean = x.mean(axis=axis, dtype=gamma.dtype)\n-        var = x.var(axis=axis, dtype=gamma.dtype) + self.eps\n-        self.std = xp.sqrt(var, dtype=var.dtype)\n+        var = x.var(axis=axis, dtype=gamma.dtype)\n+        self.std = xp.sqrt(var + self.eps, dtype=var.dtype)\n \n         running_sigma = xp.sqrt(self._running_var + self.eps,\n                                 dtype=self._running_mean.dtype)\n", "issue": "`eps` is doubly added to variance in inference of `L.BatchRenormalization`\n- `runninng_var` learns variances with `eps`\r\n- `train=False` mode uses `running_var + eps`\r\n\r\n### Conditions\r\nI tested with Chainer versions: `3.0.0`, `4.5.0`, `5.4.0`, `7.0.0a1`\r\n\r\n### Code to reproduce\r\n```python\r\nimport chainer\r\nimport numpy as np\r\nnp.random.seed(0)\r\n\r\nbrn = chainer.links.BatchRenormalization(3, eps=1.)\r\nfor _ in range(1000):\r\n    x = np.random.randn(1000, 3).astype('f')\r\n    brn(x)\r\n\r\nx = np.random.randn(1000, 3).astype('f')\r\n\r\ny = brn(x)\r\nprint(y.array.var(axis=0))\r\n\r\nwith chainer.using_config('train', False):\r\n    y = brn(x)\r\nprint(y.array.var(axis=0))\r\n```\r\n\r\n### Error messages, stack traces, or logs\r\n```\r\n[0.51281106 0.49953052 0.48790243]\r\n[0.3506052  0.33283928 0.31892547]\r\n```\r\n\r\nHere, the \"normalized\" variance is around 1/(1+eps) if train, whereas it's around 1/(1+2 eps) otherwise.\n", "before_files": [{"content": "import warnings\n\nimport numpy\n\nfrom chainer import backend\nfrom chainer.backends import cuda\nfrom chainer import configuration\nfrom chainer import function\nfrom chainer.functions.normalization import batch_normalization\nfrom chainer.utils import type_check\n\n\ndef _xhat(x, mean, std, expander):\n    x_mu = x - mean[expander]\n    x_mu /= std[expander]\n    return x_mu\n\n\nclass BatchRenormalizationFunction(function.Function):\n\n    def __init__(self, eps=2e-5, mean=None, var=None, decay=0.9,\n                 rmax=1, dmax=0, update_statistics=True):\n        self._running_mean = mean\n        self._running_var = var\n        self.rmax = rmax\n        self.dmax = dmax\n        self.r = None\n        self.update_statistics = update_statistics\n\n        self.eps = eps\n        self.decay = decay\n\n    def _warn_accessing_property(self):\n        warnings.warn(\n            'The attributes of BatchRenormalizationFunction '\n            'are deprecated. '\n            'Consider setting update_statistics=True to '\n            'batch_renormalization to update running statistics.',\n            DeprecationWarning)\n\n    @property\n    def running_mean(self):\n        self._warn_accessing_property()\n        return self._running_mean\n\n    @property\n    def running_var(self):\n        self._warn_accessing_property()\n        return self._running_var\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 3)\n        x_type, gamma_type, beta_type = in_types\n        M = type_check.eval(gamma_type.ndim)\n        type_check.expect(\n            x_type.dtype.kind == 'f',\n            x_type.ndim >= gamma_type.ndim + 1,\n            x_type.shape[1:1 + M] == gamma_type.shape,\n            # TODO(tkerola): Check shape\n            gamma_type.dtype.kind == 'f',\n            gamma_type.dtype == beta_type.dtype,\n            gamma_type.shape == beta_type.shape,\n        )\n\n    def forward(self, inputs):\n        xp = backend.get_array_module(*inputs)\n        x, gamma, beta = inputs\n\n        # Note: we must be in train mode.\n        assert configuration.config.train\n\n        head_ndim = gamma.ndim + 1\n        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)\n\n        # NOTE(tommi): cuDNN is not used since it does not support\n        # batch renormalization\n        axis = (0,) + tuple(range(head_ndim, x.ndim))\n        mean = x.mean(axis=axis, dtype=gamma.dtype)\n        var = x.var(axis=axis, dtype=gamma.dtype) + self.eps\n        self.std = xp.sqrt(var, dtype=var.dtype)\n\n        running_sigma = xp.sqrt(self._running_var + self.eps,\n                                dtype=self._running_mean.dtype)\n        self.r = xp.clip(self.std / running_sigma,\n                         1.0 / self.rmax, self.rmax)\n        d = xp.clip(\n            (mean - self._running_mean) / running_sigma,\n            -self.dmax, self.dmax)\n\n        gamma = gamma[expander]\n        beta = beta[expander]\n\n        if xp is numpy:\n            self.x_hat = _xhat(x, mean, self.std, expander)\n            self.x_hat_renorm = self.x_hat * self.r[expander] + d[expander]\n            y = gamma * self.x_hat_renorm\n            y += beta\n            y = y.astype(dtype=x.dtype)\n        else:\n            self.x_hat, self.x_hat_renorm, y = cuda.elementwise(\n                'T x, U mean, U std, U gamma, U beta, U r, U d',\n                'U x_hat, U x_hat_renorm, T y',\n                '''\n                x_hat = (x - mean) / std;\n                x_hat_renorm = x_hat * r + d;\n                y = gamma * x_hat_renorm + beta;\n                ''',\n                'brn_fwd')(\n                    x, mean[expander], self.std[expander], gamma, beta,\n                    self.r[expander], d[expander])\n\n        if self.update_statistics:\n            m = x.size // gamma[expander].size\n            self._running_mean *= self.decay\n            adjust = m / max(m - 1., 1.)  # unbiased estimation\n            temp_ar = xp.array(mean)\n            temp_ar *= (1 - self.decay)\n            self._running_mean += temp_ar\n            del temp_ar\n            self._running_var *= self.decay\n            temp_ar = xp.array(var)\n            temp_ar *= (1 - self.decay) * adjust\n            self._running_var += temp_ar\n            del temp_ar\n\n        return y,\n\n    def backward(self, inputs, grad_outputs):\n        x, gamma, _ = inputs\n        gy = grad_outputs[0]\n        head_ndim = gamma.ndim + 1\n        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)\n        m = gamma.dtype.type(x.size // gamma.size)\n        axis = (0,) + tuple(range(head_ndim, x.ndim))\n        xp = backend.get_array_module(x)\n\n        # Note: we must be in train mode.\n        assert configuration.config.train\n        # NOTE(tommi): cuDNN is not used since it does not support\n        # batch renormalization\n        gbeta = gy.sum(axis=axis, dtype=gamma.dtype)\n        ggamma = (gy * self.x_hat_renorm).sum(axis=axis)\n        gsigma_batch = (gy * self.x_hat).sum(axis=axis)\n        if xp is numpy:\n            scale = (self.r * gamma / self.std)[expander]\n            gx = scale * (gy - (self.x_hat * gsigma_batch[expander] +\n                                gbeta[expander]) / m)\n            gx = gx.astype(dtype=x.dtype)\n        else:\n            inv_m = numpy.float32(1) / m\n            gx = cuda.elementwise(\n                'T gy, U x_hat, U gamma, U std, U gsigma_batch, U gbeta, \\\n                U inv_m, U r',\n                'T gx',\n                'gx = (r * gamma / std) * (gy - (x_hat * gsigma_batch + gbeta) * \\\n                inv_m)',\n                'brn_bwd')(\n                    gy, self.x_hat, gamma[expander],\n                    self.std[expander], gsigma_batch[expander],\n                    gbeta[expander], inv_m, self.r[expander])\n        return gx, ggamma, gbeta\n\n\ndef batch_renormalization(x, gamma, beta, rmax, dmax, eps=2e-5,\n                          running_mean=None, running_var=None, decay=0.9,\n                          update_statistics=False):\n    \"\"\"Batch renormalization function.\n\n    This is an extension of batch normalization, which ensures that the\n    training and inference models generate the same outputs that depend on\n    individual examples rather than the entire minibatch.\n\n    .. note::\n\n        This function does not perform in-place update to\n        ``running_mean`` and ``running_var`` by default, contrary to\n        :func:`~chainer.functions.batch_normalization`.\n        If the function is called, it will not be possible to access the\n        updated running mean and variance statistics, because they are members\n        of the function object, which cannot be accessed by the caller.\n        If it is desired to update the running statistics, call the function\n        with ``update_statistics=True`` option.\n\n    .. note::\n\n        For the consistency with Batch Normalization, this function\n        intentionally ignores some of the theoretical flaws in Algorithm 1 of\n        the Batch Renormalization paper:\n\n        - ``F.batch_renormalization`` maintains the moving average of variances\n          :math:`\\\\sigma^2`, while the original paper maintains the moving\n          average of standard deviations :math:`\\\\sigma`.\n        - ``F.batch_renormalization`` applies Bessel's correction to update the\n          moving average of variances.\n\n    See: `Batch Renormalization: Towards Reducing Minibatch Dependence in\n    Batch-Normalized Models <https://arxiv.org/abs/1702.03275>`_\n\n    .. seealso:: :class:`~chainer.links.BatchRenormalization`\n\n    \"\"\"\n    if running_mean is None:\n        raise TypeError('running_mean is required')\n    if running_var is None:\n        raise TypeError('running_var is required')\n    return BatchRenormalizationFunction(\n        eps, running_mean, running_var, decay, rmax, dmax, update_statistics\n    )(x, gamma, beta)\n\n\ndef fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5):\n    warnings.warn(\n        'fixed_batch_renormalization is deprecated. '\n        'Use fixed_batch_normalization instead.',\n        DeprecationWarning)\n    with configuration.using_config('train', False):\n        return batch_normalization.fixed_batch_normalization(\n            x, gamma, beta, mean, var, eps\n        )\n", "path": "chainer/functions/normalization/batch_renormalization.py"}], "after_files": [{"content": "import warnings\n\nimport numpy\n\nfrom chainer import backend\nfrom chainer.backends import cuda\nfrom chainer import configuration\nfrom chainer import function\nfrom chainer.functions.normalization import batch_normalization\nfrom chainer.utils import type_check\n\n\ndef _xhat(x, mean, std, expander):\n    x_mu = x - mean[expander]\n    x_mu /= std[expander]\n    return x_mu\n\n\nclass BatchRenormalizationFunction(function.Function):\n\n    def __init__(self, eps=2e-5, mean=None, var=None, decay=0.9,\n                 rmax=1, dmax=0, update_statistics=True):\n        self._running_mean = mean\n        self._running_var = var\n        self.rmax = rmax\n        self.dmax = dmax\n        self.r = None\n        self.update_statistics = update_statistics\n\n        self.eps = eps\n        self.decay = decay\n\n    def _warn_accessing_property(self):\n        warnings.warn(\n            'The attributes of BatchRenormalizationFunction '\n            'are deprecated. '\n            'Consider setting update_statistics=True to '\n            'batch_renormalization to update running statistics.',\n            DeprecationWarning)\n\n    @property\n    def running_mean(self):\n        self._warn_accessing_property()\n        return self._running_mean\n\n    @property\n    def running_var(self):\n        self._warn_accessing_property()\n        return self._running_var\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 3)\n        x_type, gamma_type, beta_type = in_types\n        M = type_check.eval(gamma_type.ndim)\n        type_check.expect(\n            x_type.dtype.kind == 'f',\n            x_type.ndim >= gamma_type.ndim + 1,\n            x_type.shape[1:1 + M] == gamma_type.shape,\n            # TODO(tkerola): Check shape\n            gamma_type.dtype.kind == 'f',\n            gamma_type.dtype == beta_type.dtype,\n            gamma_type.shape == beta_type.shape,\n        )\n\n    def forward(self, inputs):\n        xp = backend.get_array_module(*inputs)\n        x, gamma, beta = inputs\n\n        # Note: we must be in train mode.\n        assert configuration.config.train\n\n        head_ndim = gamma.ndim + 1\n        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)\n\n        # NOTE(tommi): cuDNN is not used since it does not support\n        # batch renormalization\n        axis = (0,) + tuple(range(head_ndim, x.ndim))\n        mean = x.mean(axis=axis, dtype=gamma.dtype)\n        var = x.var(axis=axis, dtype=gamma.dtype)\n        self.std = xp.sqrt(var + self.eps, dtype=var.dtype)\n\n        running_sigma = xp.sqrt(self._running_var + self.eps,\n                                dtype=self._running_mean.dtype)\n        self.r = xp.clip(self.std / running_sigma,\n                         1.0 / self.rmax, self.rmax)\n        d = xp.clip(\n            (mean - self._running_mean) / running_sigma,\n            -self.dmax, self.dmax)\n\n        gamma = gamma[expander]\n        beta = beta[expander]\n\n        if xp is numpy:\n            self.x_hat = _xhat(x, mean, self.std, expander)\n            self.x_hat_renorm = self.x_hat * self.r[expander] + d[expander]\n            y = gamma * self.x_hat_renorm\n            y += beta\n            y = y.astype(dtype=x.dtype)\n        else:\n            self.x_hat, self.x_hat_renorm, y = cuda.elementwise(\n                'T x, U mean, U std, U gamma, U beta, U r, U d',\n                'U x_hat, U x_hat_renorm, T y',\n                '''\n                x_hat = (x - mean) / std;\n                x_hat_renorm = x_hat * r + d;\n                y = gamma * x_hat_renorm + beta;\n                ''',\n                'brn_fwd')(\n                    x, mean[expander], self.std[expander], gamma, beta,\n                    self.r[expander], d[expander])\n\n        if self.update_statistics:\n            m = x.size // gamma[expander].size\n            self._running_mean *= self.decay\n            adjust = m / max(m - 1., 1.)  # unbiased estimation\n            temp_ar = xp.array(mean)\n            temp_ar *= (1 - self.decay)\n            self._running_mean += temp_ar\n            del temp_ar\n            self._running_var *= self.decay\n            temp_ar = xp.array(var)\n            temp_ar *= (1 - self.decay) * adjust\n            self._running_var += temp_ar\n            del temp_ar\n\n        return y,\n\n    def backward(self, inputs, grad_outputs):\n        x, gamma, _ = inputs\n        gy = grad_outputs[0]\n        head_ndim = gamma.ndim + 1\n        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)\n        m = gamma.dtype.type(x.size // gamma.size)\n        axis = (0,) + tuple(range(head_ndim, x.ndim))\n        xp = backend.get_array_module(x)\n\n        # Note: we must be in train mode.\n        assert configuration.config.train\n        # NOTE(tommi): cuDNN is not used since it does not support\n        # batch renormalization\n        gbeta = gy.sum(axis=axis, dtype=gamma.dtype)\n        ggamma = (gy * self.x_hat_renorm).sum(axis=axis)\n        gsigma_batch = (gy * self.x_hat).sum(axis=axis)\n        if xp is numpy:\n            scale = (self.r * gamma / self.std)[expander]\n            gx = scale * (gy - (self.x_hat * gsigma_batch[expander] +\n                                gbeta[expander]) / m)\n            gx = gx.astype(dtype=x.dtype)\n        else:\n            inv_m = numpy.float32(1) / m\n            gx = cuda.elementwise(\n                'T gy, U x_hat, U gamma, U std, U gsigma_batch, U gbeta, \\\n                U inv_m, U r',\n                'T gx',\n                'gx = (r * gamma / std) * (gy - (x_hat * gsigma_batch + gbeta) * \\\n                inv_m)',\n                'brn_bwd')(\n                    gy, self.x_hat, gamma[expander],\n                    self.std[expander], gsigma_batch[expander],\n                    gbeta[expander], inv_m, self.r[expander])\n        return gx, ggamma, gbeta\n\n\ndef batch_renormalization(x, gamma, beta, rmax, dmax, eps=2e-5,\n                          running_mean=None, running_var=None, decay=0.9,\n                          update_statistics=False):\n    \"\"\"Batch renormalization function.\n\n    This is an extension of batch normalization, which ensures that the\n    training and inference models generate the same outputs that depend on\n    individual examples rather than the entire minibatch.\n\n    .. note::\n\n        This function does not perform in-place update to\n        ``running_mean`` and ``running_var`` by default, contrary to\n        :func:`~chainer.functions.batch_normalization`.\n        If the function is called, it will not be possible to access the\n        updated running mean and variance statistics, because they are members\n        of the function object, which cannot be accessed by the caller.\n        If it is desired to update the running statistics, call the function\n        with ``update_statistics=True`` option.\n\n    .. note::\n\n        For the consistency with Batch Normalization, this function\n        intentionally ignores some of the theoretical flaws in Algorithm 1 of\n        the Batch Renormalization paper:\n\n        - ``F.batch_renormalization`` maintains the moving average of variances\n          :math:`\\\\sigma^2`, while the original paper maintains the moving\n          average of standard deviations :math:`\\\\sigma`.\n        - ``F.batch_renormalization`` applies Bessel's correction to update the\n          moving average of variances.\n\n    See: `Batch Renormalization: Towards Reducing Minibatch Dependence in\n    Batch-Normalized Models <https://arxiv.org/abs/1702.03275>`_\n\n    .. seealso:: :class:`~chainer.links.BatchRenormalization`\n\n    \"\"\"\n    if running_mean is None:\n        raise TypeError('running_mean is required')\n    if running_var is None:\n        raise TypeError('running_var is required')\n    return BatchRenormalizationFunction(\n        eps, running_mean, running_var, decay, rmax, dmax, update_statistics\n    )(x, gamma, beta)\n\n\ndef fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5):\n    warnings.warn(\n        'fixed_batch_renormalization is deprecated. '\n        'Use fixed_batch_normalization instead.',\n        DeprecationWarning)\n    with configuration.using_config('train', False):\n        return batch_normalization.fixed_batch_normalization(\n            x, gamma, beta, mean, var, eps\n        )\n", "path": "chainer/functions/normalization/batch_renormalization.py"}]}