We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Wrong default for horovod.tensorflow.keras.allreduce(average...
In Horovod 0.21.1 the default for `average` in `allreduce` is still `True` leading to 

> ValueError: The op parameter supersedes average. Please provide only one of them.

when using `op=...` (only).

This is only in in `horovod.tensorflow.keras`, not in `horovod.tensorflow`

BTW: In TF2, is there any benefit of using `horovod.tensorflow.keras` over `horovod.tensorflow` when not disabling eager execution (which in my tests is pretty much unfeasible)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `horovod/tensorflow/keras/__init__.py`
Content:
```
1 # Copyright 2018 Uber Technologies, Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 # ==============================================================================
15 
16 import inspect
17 
18 import tensorflow as tf
19 
20 from tensorflow import keras
21 from tensorflow.python.keras import backend as K
22 
23 from horovod.tensorflow import init
24 from horovod.tensorflow import shutdown
25 from horovod.tensorflow import is_initialized, start_timeline, stop_timeline
26 from horovod.tensorflow import size
27 from horovod.tensorflow import local_size
28 from horovod.tensorflow import rank
29 from horovod.tensorflow import local_rank
30 from horovod.tensorflow import mpi_threads_supported, mpi_enabled, mpi_built
31 from horovod.tensorflow import gloo_enabled, gloo_built
32 from horovod.tensorflow import nccl_built, ddl_built, ccl_built, cuda_built, rocm_built
33 from horovod.tensorflow import Average, Compression, Sum
34 
35 import horovod._keras as _impl
36 from horovod.tensorflow.keras import callbacks, elastic
37 
38 
39 try:
40     # In later versions of TensorFlow, optimizers are spread across multiple modules. This set is used to distinguish
41     # stock optimizers that come with tf.keras from custom optimizers that may need to be wrapped specially.
42     _OPTIMIZER_MODULES = set([obj.__module__ for name, obj in inspect.getmembers(tf.keras.optimizers)
43                               if isinstance(obj, type(tf.keras.optimizers.Optimizer))])
44 except:
45     _OPTIMIZER_MODULES = set()
46 
47 
48 def DistributedOptimizer(optimizer, name=None,
49                          device_dense='', device_sparse='',
50                          compression=Compression.none,
51                          sparse_as_dense=False,
52                          gradient_predivide_factor=1.0,
53                          op=Average,
54                          backward_passes_per_step=1,
55                          average_aggregated_gradients=False):
56     """
57     An optimizer that wraps another keras.optimizers.Optimizer, using an allreduce to
58     average gradient values before applying gradients to model weights.
59 
60     Args:
61         optimizer: Optimizer to use for computing gradients and applying updates.
62         name: Optional name prefix for the operations created when applying
63               gradients. Defaults to "Distributed" followed by the provided
64               optimizer type.
65         device_dense: Device to be used for dense tensors. Uses GPU by default
66                       if Horovod was build with HOROVOD_GPU_OPERATIONS.
67         device_sparse: Device to be used for sparse tensors. Uses GPU by default
68                        if Horovod was build with HOROVOD_GPU_OPERATIONS.
69         compression: Compression algorithm used to reduce the amount of data
70                      sent and received by each worker node.  Defaults to not
71                      using compression.
72         sparse_as_dense: Treat all sparse gradients as dense tensors.  This can
73                          help improve performance and memory utilization if
74                          the original sparse gradient has high density.
75                          Defaults to false.
76         gradient_predivide_factor: gradient_predivide_factor splits the averaging
77                                    before and after the sum. Gradients are scaled by
78                                    1.0 / gradient_predivide_factor before the sum and
79                                    gradient_predivide_factor / size after the sum.
80         op: The reduction operation to use when combining gradients across
81             different ranks. Defaults to Average.
82         backward_passes_per_step: Number of backward passes to perform before calling
83                                   hvd.allreduce. This allows accumulating updates over
84                                   multiple mini-batches before reducing and applying them.
85         average_aggregated_gradients: Whether to average the aggregated gradients that
86                                       have been accumulated over multiple mini-batches.
87                                       If true divides gradient updates by
88                                       backward_passes_per_step.
89                                       Only applicable for backward_passes_per_step > 1.
90     """
91     if gradient_predivide_factor != 1.0 and rocm_built():
92             raise ValueError('gradient_predivide_factor not supported yet with ROCm')
93 
94     if op != Average and op != Sum:
95         raise ValueError('op currently only supports Average and Sum')
96 
97     return _impl.create_distributed_optimizer(
98         keras=keras,
99         optimizer=optimizer,
100         name=name,
101         device_dense=device_dense,
102         device_sparse=device_sparse,
103         compression=compression,
104         sparse_as_dense=sparse_as_dense,
105         gradient_predivide_factor=gradient_predivide_factor,
106         op=op,
107         backward_passes_per_step=backward_passes_per_step,
108         average_aggregated_gradients=average_aggregated_gradients,
109     )
110 
111 
112 def broadcast_global_variables(root_rank):
113     """Broadcasts all global variables from root rank to all other processes.
114 
115     Arguments:
116         root_rank: Rank of the process from which global variables will be broadcasted
117                    to all other processes.
118     """
119     return _impl.broadcast_global_variables(K, root_rank)
120 
121 
122 def allreduce(value, name=None, average=True,
123               prescale_factor=1.0,
124               postscale_factor=1.0,
125               op=None,
126               compression=Compression.none):
127     """
128     Perform an allreduce on a tensor-compatible value.
129 
130     Arguments:
131         value: A tensor-compatible value to reduce.
132                The shape of the input must be identical across all ranks.
133         name: Optional name for the constants created by this operation.
134         average:
135             .. warning:: .. deprecated:: 0.19.0
136 
137                Use `op` instead. Will be removed in v0.21.0.
138 
139         prescale_factor: Multiplicative factor to scale tensor before allreduce.
140         postscale_factor: Multiplicative factor to scale tensor after allreduce.
141         op: The reduction operation to combine tensors across different ranks.
142             Defaults to Average if None is given.
143         compression: Compression algorithm used to reduce the amount of data
144                      sent and received by each worker node.  Defaults to not
145                      using compression.
146     """
147     return _impl.allreduce(
148         backend=K,
149         value=value,
150         name=name,
151         average=average,
152         prescale_factor=prescale_factor,
153         postscale_factor=postscale_factor,
154         op=op,
155         compression=compression)
156 
157 
158 def allgather(value, name=None):
159     """
160     Perform an allgather on a tensor-compatible value.
161 
162     The concatenation is done on the first dimension, so the input values on the
163     different processes must have the same rank and shape, except for the first
164     dimension, which is allowed to be different.
165 
166     Arguments:
167         value: A tensor-compatible value to gather.
168         name: Optional name prefix for the constants created by this operation.
169     """
170     return _impl.allgather(K, value, name)
171 
172 
173 def broadcast(value, root_rank, name=None):
174     """
175     Perform a broadcast on a tensor-compatible value.
176 
177     Arguments:
178         value: A tensor-compatible value to reduce.
179                The shape of the input must be identical across all ranks.
180         root_rank: Rank of the process from which global variables will be
181                    broadcasted to all other processes.
182         name: Optional name for the constants created by this operation.
183     """
184     return _impl.broadcast(K, value, root_rank, name)
185 
186 
187 def load_model(filepath, custom_optimizers=None, custom_objects=None, compression=Compression.none):
188     """
189     Loads a saved Keras model with a Horovod DistributedOptimizer.
190 
191     The DistributedOptimizer will wrap the underlying optimizer used to train
192     the saved model, so that the optimizer state (params and weights) will
193     be picked up for retraining.
194 
195     By default, all optimizers in the module `keras.optimizers` will be loaded
196     and wrapped without needing to specify any `custom_optimizers` or
197     `custom_objects`.
198 
199     Arguments:
200         filepath: One of the following:
201             - string, path to the saved model, or
202             - h5py.File object from which to load the model
203         custom_optimizers: Optional list of Optimizer subclasses to support
204             during loading.
205         custom_objects: Optional dictionary mapping names (strings) to custom
206             classes or functions to be considered during deserialization.
207         compression: Compression algorithm used to reduce the amount of data
208                      sent and received by each worker node.  Defaults to not
209                      using compression.
210 
211     Returns:
212         A Keras model instance.
213 
214     Raises:
215         ImportError: If h5py is not available.
216         ValueError: In case of an invalid savefile.
217     """
218     def wrap_optimizer(cls):
219         return lambda **kwargs: DistributedOptimizer(cls(**kwargs), compression=compression)
220     return _impl.load_model(keras, wrap_optimizer, _OPTIMIZER_MODULES, filepath, custom_optimizers, custom_objects)
221 
222
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/horovod/tensorflow/keras/__init__.py b/horovod/tensorflow/keras/__init__.py
--- a/horovod/tensorflow/keras/__init__.py
+++ b/horovod/tensorflow/keras/__init__.py
@@ -119,7 +119,7 @@
     return _impl.broadcast_global_variables(K, root_rank)
 
 
-def allreduce(value, name=None, average=True,
+def allreduce(value, name=None, average=None,
               prescale_factor=1.0,
               postscale_factor=1.0,
               op=None,

{"golden_diff": "diff --git a/horovod/tensorflow/keras/__init__.py b/horovod/tensorflow/keras/__init__.py\n--- a/horovod/tensorflow/keras/__init__.py\n+++ b/horovod/tensorflow/keras/__init__.py\n@@ -119,7 +119,7 @@\n     return _impl.broadcast_global_variables(K, root_rank)\n \n \n-def allreduce(value, name=None, average=True,\n+def allreduce(value, name=None, average=None,\n               prescale_factor=1.0,\n               postscale_factor=1.0,\n               op=None,\n", "issue": "Wrong default for horovod.tensorflow.keras.allreduce(average...\nIn Horovod 0.21.1 the default for `average` in `allreduce` is still `True` leading to \r\n\r\n> ValueError: The op parameter supersedes average. Please provide only one of them.\r\n\r\nwhen using `op=...` (only).\r\n\r\nThis is only in in `horovod.tensorflow.keras`, not in `horovod.tensorflow`\r\n\r\nBTW: In TF2, is there any benefit of using `horovod.tensorflow.keras` over `horovod.tensorflow` when not disabling eager execution (which in my tests is pretty much unfeasible)\n", "before_files": [{"content": "# Copyright 2018 Uber Technologies, Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\nimport inspect\n\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.python.keras import backend as K\n\nfrom horovod.tensorflow import init\nfrom horovod.tensorflow import shutdown\nfrom horovod.tensorflow import is_initialized, start_timeline, stop_timeline\nfrom horovod.tensorflow import size\nfrom horovod.tensorflow import local_size\nfrom horovod.tensorflow import rank\nfrom horovod.tensorflow import local_rank\nfrom horovod.tensorflow import mpi_threads_supported, mpi_enabled, mpi_built\nfrom horovod.tensorflow import gloo_enabled, gloo_built\nfrom horovod.tensorflow import nccl_built, ddl_built, ccl_built, cuda_built, rocm_built\nfrom horovod.tensorflow import Average, Compression, Sum\n\nimport horovod._keras as _impl\nfrom horovod.tensorflow.keras import callbacks, elastic\n\n\ntry:\n    # In later versions of TensorFlow, optimizers are spread across multiple modules. This set is used to distinguish\n    # stock optimizers that come with tf.keras from custom optimizers that may need to be wrapped specially.\n    _OPTIMIZER_MODULES = set([obj.__module__ for name, obj in inspect.getmembers(tf.keras.optimizers)\n                              if isinstance(obj, type(tf.keras.optimizers.Optimizer))])\nexcept:\n    _OPTIMIZER_MODULES = set()\n\n\ndef DistributedOptimizer(optimizer, name=None,\n                         device_dense='', device_sparse='',\n                         compression=Compression.none,\n                         sparse_as_dense=False,\n                         gradient_predivide_factor=1.0,\n                         op=Average,\n                         backward_passes_per_step=1,\n                         average_aggregated_gradients=False):\n    \"\"\"\n    An optimizer that wraps another keras.optimizers.Optimizer, using an allreduce to\n    average gradient values before applying gradients to model weights.\n\n    Args:\n        optimizer: Optimizer to use for computing gradients and applying updates.\n        name: Optional name prefix for the operations created when applying\n              gradients. Defaults to \"Distributed\" followed by the provided\n              optimizer type.\n        device_dense: Device to be used for dense tensors. Uses GPU by default\n                      if Horovod was build with HOROVOD_GPU_OPERATIONS.\n        device_sparse: Device to be used for sparse tensors. Uses GPU by default\n                       if Horovod was build with HOROVOD_GPU_OPERATIONS.\n        compression: Compression algorithm used to reduce the amount of data\n                     sent and received by each worker node.  Defaults to not\n                     using compression.\n        sparse_as_dense: Treat all sparse gradients as dense tensors.  This can\n                         help improve performance and memory utilization if\n                         the original sparse gradient has high density.\n                         Defaults to false.\n        gradient_predivide_factor: gradient_predivide_factor splits the averaging\n                                   before and after the sum. Gradients are scaled by\n                                   1.0 / gradient_predivide_factor before the sum and\n                                   gradient_predivide_factor / size after the sum.\n        op: The reduction operation to use when combining gradients across\n            different ranks. Defaults to Average.\n        backward_passes_per_step: Number of backward passes to perform before calling\n                                  hvd.allreduce. This allows accumulating updates over\n                                  multiple mini-batches before reducing and applying them.\n        average_aggregated_gradients: Whether to average the aggregated gradients that\n                                      have been accumulated over multiple mini-batches.\n                                      If true divides gradient updates by\n                                      backward_passes_per_step.\n                                      Only applicable for backward_passes_per_step > 1.\n    \"\"\"\n    if gradient_predivide_factor != 1.0 and rocm_built():\n            raise ValueError('gradient_predivide_factor not supported yet with ROCm')\n\n    if op != Average and op != Sum:\n        raise ValueError('op currently only supports Average and Sum')\n\n    return _impl.create_distributed_optimizer(\n        keras=keras,\n        optimizer=optimizer,\n        name=name,\n        device_dense=device_dense,\n        device_sparse=device_sparse,\n        compression=compression,\n        sparse_as_dense=sparse_as_dense,\n        gradient_predivide_factor=gradient_predivide_factor,\n        op=op,\n        backward_passes_per_step=backward_passes_per_step,\n        average_aggregated_gradients=average_aggregated_gradients,\n    )\n\n\ndef broadcast_global_variables(root_rank):\n    \"\"\"Broadcasts all global variables from root rank to all other processes.\n\n    Arguments:\n        root_rank: Rank of the process from which global variables will be broadcasted\n                   to all other processes.\n    \"\"\"\n    return _impl.broadcast_global_variables(K, root_rank)\n\n\ndef allreduce(value, name=None, average=True,\n              prescale_factor=1.0,\n              postscale_factor=1.0,\n              op=None,\n              compression=Compression.none):\n    \"\"\"\n    Perform an allreduce on a tensor-compatible value.\n\n    Arguments:\n        value: A tensor-compatible value to reduce.\n               The shape of the input must be identical across all ranks.\n        name: Optional name for the constants created by this operation.\n        average:\n            .. warning:: .. deprecated:: 0.19.0\n\n               Use `op` instead. Will be removed in v0.21.0.\n\n        prescale_factor: Multiplicative factor to scale tensor before allreduce.\n        postscale_factor: Multiplicative factor to scale tensor after allreduce.\n        op: The reduction operation to combine tensors across different ranks.\n            Defaults to Average if None is given.\n        compression: Compression algorithm used to reduce the amount of data\n                     sent and received by each worker node.  Defaults to not\n                     using compression.\n    \"\"\"\n    return _impl.allreduce(\n        backend=K,\n        value=value,\n        name=name,\n        average=average,\n        prescale_factor=prescale_factor,\n        postscale_factor=postscale_factor,\n        op=op,\n        compression=compression)\n\n\ndef allgather(value, name=None):\n    \"\"\"\n    Perform an allgather on a tensor-compatible value.\n\n    The concatenation is done on the first dimension, so the input values on the\n    different processes must have the same rank and shape, except for the first\n    dimension, which is allowed to be different.\n\n    Arguments:\n        value: A tensor-compatible value to gather.\n        name: Optional name prefix for the constants created by this operation.\n    \"\"\"\n    return _impl.allgather(K, value, name)\n\n\ndef broadcast(value, root_rank, name=None):\n    \"\"\"\n    Perform a broadcast on a tensor-compatible value.\n\n    Arguments:\n        value: A tensor-compatible value to reduce.\n               The shape of the input must be identical across all ranks.\n        root_rank: Rank of the process from which global variables will be\n                   broadcasted to all other processes.\n        name: Optional name for the constants created by this operation.\n    \"\"\"\n    return _impl.broadcast(K, value, root_rank, name)\n\n\ndef load_model(filepath, custom_optimizers=None, custom_objects=None, compression=Compression.none):\n    \"\"\"\n    Loads a saved Keras model with a Horovod DistributedOptimizer.\n\n    The DistributedOptimizer will wrap the underlying optimizer used to train\n    the saved model, so that the optimizer state (params and weights) will\n    be picked up for retraining.\n\n    By default, all optimizers in the module `keras.optimizers` will be loaded\n    and wrapped without needing to specify any `custom_optimizers` or\n    `custom_objects`.\n\n    Arguments:\n        filepath: One of the following:\n            - string, path to the saved model, or\n            - h5py.File object from which to load the model\n        custom_optimizers: Optional list of Optimizer subclasses to support\n            during loading.\n        custom_objects: Optional dictionary mapping names (strings) to custom\n            classes or functions to be considered during deserialization.\n        compression: Compression algorithm used to reduce the amount of data\n                     sent and received by each worker node.  Defaults to not\n                     using compression.\n\n    Returns:\n        A Keras model instance.\n\n    Raises:\n        ImportError: If h5py is not available.\n        ValueError: In case of an invalid savefile.\n    \"\"\"\n    def wrap_optimizer(cls):\n        return lambda **kwargs: DistributedOptimizer(cls(**kwargs), compression=compression)\n    return _impl.load_model(keras, wrap_optimizer, _OPTIMIZER_MODULES, filepath, custom_optimizers, custom_objects)\n\n", "path": "horovod/tensorflow/keras/__init__.py"}], "after_files": [{"content": "# Copyright 2018 Uber Technologies, Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\nimport inspect\n\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.python.keras import backend as K\n\nfrom horovod.tensorflow import init\nfrom horovod.tensorflow import shutdown\nfrom horovod.tensorflow import is_initialized, start_timeline, stop_timeline\nfrom horovod.tensorflow import size\nfrom horovod.tensorflow import local_size\nfrom horovod.tensorflow import rank\nfrom horovod.tensorflow import local_rank\nfrom horovod.tensorflow import mpi_threads_supported, mpi_enabled, mpi_built\nfrom horovod.tensorflow import gloo_enabled, gloo_built\nfrom horovod.tensorflow import nccl_built, ddl_built, ccl_built, cuda_built, rocm_built\nfrom horovod.tensorflow import Average, Compression, Sum\n\nimport horovod._keras as _impl\nfrom horovod.tensorflow.keras import callbacks, elastic\n\n\ntry:\n    # In later versions of TensorFlow, optimizers are spread across multiple modules. This set is used to distinguish\n    # stock optimizers that come with tf.keras from custom optimizers that may need to be wrapped specially.\n    _OPTIMIZER_MODULES = set([obj.__module__ for name, obj in inspect.getmembers(tf.keras.optimizers)\n                              if isinstance(obj, type(tf.keras.optimizers.Optimizer))])\nexcept:\n    _OPTIMIZER_MODULES = set()\n\n\ndef DistributedOptimizer(optimizer, name=None,\n                         device_dense='', device_sparse='',\n                         compression=Compression.none,\n                         sparse_as_dense=False,\n                         gradient_predivide_factor=1.0,\n                         op=Average,\n                         backward_passes_per_step=1,\n                         average_aggregated_gradients=False):\n    \"\"\"\n    An optimizer that wraps another keras.optimizers.Optimizer, using an allreduce to\n    average gradient values before applying gradients to model weights.\n\n    Args:\n        optimizer: Optimizer to use for computing gradients and applying updates.\n        name: Optional name prefix for the operations created when applying\n              gradients. Defaults to \"Distributed\" followed by the provided\n              optimizer type.\n        device_dense: Device to be used for dense tensors. Uses GPU by default\n                      if Horovod was build with HOROVOD_GPU_OPERATIONS.\n        device_sparse: Device to be used for sparse tensors. Uses GPU by default\n                       if Horovod was build with HOROVOD_GPU_OPERATIONS.\n        compression: Compression algorithm used to reduce the amount of data\n                     sent and received by each worker node.  Defaults to not\n                     using compression.\n        sparse_as_dense: Treat all sparse gradients as dense tensors.  This can\n                         help improve performance and memory utilization if\n                         the original sparse gradient has high density.\n                         Defaults to false.\n        gradient_predivide_factor: gradient_predivide_factor splits the averaging\n                                   before and after the sum. Gradients are scaled by\n                                   1.0 / gradient_predivide_factor before the sum and\n                                   gradient_predivide_factor / size after the sum.\n        op: The reduction operation to use when combining gradients across\n            different ranks. Defaults to Average.\n        backward_passes_per_step: Number of backward passes to perform before calling\n                                  hvd.allreduce. This allows accumulating updates over\n                                  multiple mini-batches before reducing and applying them.\n        average_aggregated_gradients: Whether to average the aggregated gradients that\n                                      have been accumulated over multiple mini-batches.\n                                      If true divides gradient updates by\n                                      backward_passes_per_step.\n                                      Only applicable for backward_passes_per_step > 1.\n    \"\"\"\n    if gradient_predivide_factor != 1.0 and rocm_built():\n            raise ValueError('gradient_predivide_factor not supported yet with ROCm')\n\n    if op != Average and op != Sum:\n        raise ValueError('op currently only supports Average and Sum')\n\n    return _impl.create_distributed_optimizer(\n        keras=keras,\n        optimizer=optimizer,\n        name=name,\n        device_dense=device_dense,\n        device_sparse=device_sparse,\n        compression=compression,\n        sparse_as_dense=sparse_as_dense,\n        gradient_predivide_factor=gradient_predivide_factor,\n        op=op,\n        backward_passes_per_step=backward_passes_per_step,\n        average_aggregated_gradients=average_aggregated_gradients,\n    )\n\n\ndef broadcast_global_variables(root_rank):\n    \"\"\"Broadcasts all global variables from root rank to all other processes.\n\n    Arguments:\n        root_rank: Rank of the process from which global variables will be broadcasted\n                   to all other processes.\n    \"\"\"\n    return _impl.broadcast_global_variables(K, root_rank)\n\n\ndef allreduce(value, name=None, average=None,\n              prescale_factor=1.0,\n              postscale_factor=1.0,\n              op=None,\n              compression=Compression.none):\n    \"\"\"\n    Perform an allreduce on a tensor-compatible value.\n\n    Arguments:\n        value: A tensor-compatible value to reduce.\n               The shape of the input must be identical across all ranks.\n        name: Optional name for the constants created by this operation.\n        average:\n            .. warning:: .. deprecated:: 0.19.0\n\n               Use `op` instead. Will be removed in v0.21.0.\n\n        prescale_factor: Multiplicative factor to scale tensor before allreduce.\n        postscale_factor: Multiplicative factor to scale tensor after allreduce.\n        op: The reduction operation to combine tensors across different ranks.\n            Defaults to Average if None is given.\n        compression: Compression algorithm used to reduce the amount of data\n                     sent and received by each worker node.  Defaults to not\n                     using compression.\n    \"\"\"\n    return _impl.allreduce(\n        backend=K,\n        value=value,\n        name=name,\n        average=average,\n        prescale_factor=prescale_factor,\n        postscale_factor=postscale_factor,\n        op=op,\n        compression=compression)\n\n\ndef allgather(value, name=None):\n    \"\"\"\n    Perform an allgather on a tensor-compatible value.\n\n    The concatenation is done on the first dimension, so the input values on the\n    different processes must have the same rank and shape, except for the first\n    dimension, which is allowed to be different.\n\n    Arguments:\n        value: A tensor-compatible value to gather.\n        name: Optional name prefix for the constants created by this operation.\n    \"\"\"\n    return _impl.allgather(K, value, name)\n\n\ndef broadcast(value, root_rank, name=None):\n    \"\"\"\n    Perform a broadcast on a tensor-compatible value.\n\n    Arguments:\n        value: A tensor-compatible value to reduce.\n               The shape of the input must be identical across all ranks.\n        root_rank: Rank of the process from which global variables will be\n                   broadcasted to all other processes.\n        name: Optional name for the constants created by this operation.\n    \"\"\"\n    return _impl.broadcast(K, value, root_rank, name)\n\n\ndef load_model(filepath, custom_optimizers=None, custom_objects=None, compression=Compression.none):\n    \"\"\"\n    Loads a saved Keras model with a Horovod DistributedOptimizer.\n\n    The DistributedOptimizer will wrap the underlying optimizer used to train\n    the saved model, so that the optimizer state (params and weights) will\n    be picked up for retraining.\n\n    By default, all optimizers in the module `keras.optimizers` will be loaded\n    and wrapped without needing to specify any `custom_optimizers` or\n    `custom_objects`.\n\n    Arguments:\n        filepath: One of the following:\n            - string, path to the saved model, or\n            - h5py.File object from which to load the model\n        custom_optimizers: Optional list of Optimizer subclasses to support\n            during loading.\n        custom_objects: Optional dictionary mapping names (strings) to custom\n            classes or functions to be considered during deserialization.\n        compression: Compression algorithm used to reduce the amount of data\n                     sent and received by each worker node.  Defaults to not\n                     using compression.\n\n    Returns:\n        A Keras model instance.\n\n    Raises:\n        ImportError: If h5py is not available.\n        ValueError: In case of an invalid savefile.\n    \"\"\"\n    def wrap_optimizer(cls):\n        return lambda **kwargs: DistributedOptimizer(cls(**kwargs), compression=compression)\n    return _impl.load_model(keras, wrap_optimizer, _OPTIMIZER_MODULES, filepath, custom_optimizers, custom_objects)\n\n", "path": "horovod/tensorflow/keras/__init__.py"}]}