We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Spark tests fail with AttributeError: 'pyarrow.lib.Schema' object has no attribute 'to_arrow_schema'
Spark tests fail with

```
Traceback (most recent call last):
  File "/horovod/examples/spark/pytorch/pytorch_spark_mnist.py", line 122, in <module>
    torch_model = torch_estimator.fit(train_df).setOutputCols(['label_prob'])
  File "/usr/local/lib/python3.8/dist-packages/horovod/spark/common/estimator.py", line 35, in fit
    return super(HorovodEstimator, self).fit(df, params)
  File "/usr/local/lib/python3.8/dist-packages/pyspark/ml/base.py", line 205, in fit
    return self._fit(dataset)
  File "/usr/local/lib/python3.8/dist-packages/horovod/spark/common/estimator.py", line 68, in _fit
    with util.prepare_data(backend.num_processes(),
  File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__
    return next(self.gen)
  File "/usr/local/lib/python3.8/dist-packages/horovod/spark/common/util.py", line 735, in prepare_data
    dataset_idx = _get_or_create_dataset(key, store, df, feature_columns, label_columns,
  File "/usr/local/lib/python3.8/dist-packages/horovod/spark/common/util.py", line 672, in _get_or_create_dataset
    train_rows, val_rows, pq_metadata, avg_row_size = get_simple_meta_from_parquet(
  File "/usr/local/lib/python3.8/dist-packages/horovod/spark/common/util.py", line 495, in get_simple_meta_from_parquet
    train_data_schema = train_data.schema.to_arrow_schema()
AttributeError: 'pyarrow.lib.Schema' object has no attribute 'to_arrow_schema'
```
https://github.com/horovod/horovod/actions/runs/4025636527/jobs/6919132641

It's probably related to the 11.0 release of pyarrow. https://pypi.org/project/pyarrow/#history

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `setup.py`
Content:
```
1 # Copyright 2019 Uber Technologies, Inc. All Rights Reserved.
2 # Modifications copyright Microsoft
3 # Modifications copyright (C) 2020, NVIDIA CORPORATION. All rights reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 # ==============================================================================
17 
18 import atexit
19 import io
20 import os
21 import re
22 import shutil
23 import subprocess
24 import sys
25 import tempfile
26 import textwrap
27 
28 from setuptools import setup, Extension, find_packages
29 from setuptools.command.build_ext import build_ext
30 
31 from horovod import __version__
32 
33 _FRAMEWORK_METADATA_FILE = 'horovod/metadata.json'
34 
35 class CMakeExtension(Extension):
36     def __init__(self, name, cmake_lists_dir='.', sources=None, **kwa):
37         if sources is None:
38             sources = []
39         Extension.__init__(self, name, sources=sources, **kwa)
40         self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
41 
42 
43 tensorflow_mpi_lib = CMakeExtension('horovod.tensorflow.mpi_lib',
44                                      cmake_lists_dir='.', sources=[])
45 torch_mpi_lib_v2 = CMakeExtension('horovod.torch.mpi_lib_v2',
46                                      cmake_lists_dir='.', sources=[])
47 mxnet_mpi_lib = CMakeExtension('horovod.mxnet.mpi_lib',
48                                      cmake_lists_dir='.', sources=[])
49 
50 def is_build_action():
51     if len(sys.argv) <= 1:
52         return False
53 
54     if sys.argv[1].startswith('build'):
55         return True
56 
57     if sys.argv[1].startswith('bdist'):
58         return True
59 
60     if sys.argv[1].startswith('install'):
61         return True
62 
63     if sys.argv[1].startswith('develop'):
64         return True
65 
66 def get_cmake_bin():
67     from packaging import version
68 
69     if 'HOROVOD_CMAKE' in os.environ:
70         return os.environ['HOROVOD_CMAKE']
71 
72     cmake_bin = 'cmake'
73     try:
74         out = subprocess.check_output([cmake_bin, '--version'])
75     except OSError:
76         cmake_installed_version = version.parse("0.0")
77     else:
78         cmake_installed_version = version.parse(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
79 
80     if cmake_installed_version < version.parse("3.13.0"):
81         print("Could not find a recent CMake to build Horovod. "
82               "Attempting to install CMake 3.13 to a temporary location via pip.", flush=True)
83         cmake_temp_dir = tempfile.TemporaryDirectory(prefix="horovod-cmake-tmp")
84         atexit.register(cmake_temp_dir.cleanup)
85         try:
86             _ = subprocess.check_output(["pip", "install", "--target", cmake_temp_dir.name, "cmake~=3.13.0"])
87         except Exception:
88             raise RuntimeError("Failed to install temporary CMake. "
89                                "Please update your CMake to 3.13+ or set HOROVOD_CMAKE appropriately.")
90         cmake_bin = os.path.join(cmake_temp_dir.name, "bin", "run_cmake")
91         with io.open(cmake_bin, "w") as f_run_cmake:
92             f_run_cmake.write(
93                 f"#!/bin/sh\nPYTHONPATH={cmake_temp_dir.name} {os.path.join(cmake_temp_dir.name, 'bin', 'cmake')} \"$@\"")
94         os.chmod(cmake_bin, 0o755)
95 
96     return cmake_bin
97 
98 
99 class custom_build_ext(build_ext):
100     def build_extensions(self):
101         if os.getenv('HOROVOD_SKIP_COMPILE') == '1':
102             # Skip building extensions using CMake
103             print("Horovod is being installed without native libraries")
104             return
105 
106         cmake_bin = get_cmake_bin()
107 
108         config = 'Debug' if self.debug or os.environ.get('HOROVOD_DEBUG') == "1" else 'RelWithDebInfo'
109 
110         ext_name = self.extensions[0].name
111         build_dir = self.get_ext_fullpath(ext_name).replace(self.get_ext_filename(ext_name), '')
112         build_dir = os.path.abspath(build_dir)
113 
114         cmake_args = ['-DCMAKE_BUILD_TYPE=' + config,
115                       '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(config.upper(), build_dir),
116                       '-DPYTHON_EXECUTABLE:FILEPATH=' + sys.executable]
117 
118         make_args = ['-j8'] if not os.environ.get('MAKEFLAGS') else []
119         if self.verbose:
120             make_args.append('VERBOSE=1')
121 
122         cmake_build_args = ['--config', config]
123         if make_args:
124             # -- specifies that these args are going to the native build tool: make
125             cmake_build_args += ['--'] + make_args
126 
127         cmake_build_dir = os.path.join(self.build_temp, config)
128         if not os.path.exists(cmake_build_dir):
129             os.makedirs(cmake_build_dir)
130 
131         config_and_build_commands = [
132             [cmake_bin, self.extensions[0].cmake_lists_dir] + cmake_args,
133             [cmake_bin, '--build', '.'] + cmake_build_args
134         ]
135 
136         if self.verbose:
137             print(f"Running CMake in {cmake_build_dir}:")
138             for command in config_and_build_commands:
139                 print(" ".join(command))
140             sys.stdout.flush()
141 
142         # Config and build the extension
143         try:
144             for command in config_and_build_commands:
145                 subprocess.check_call(command, cwd=cmake_build_dir)
146         except OSError as e:
147             raise RuntimeError('CMake failed: {}'.format(str(e)))
148 
149         if sys.argv[1].startswith('develop'):
150             # Copy over metadata.json file from build directory
151             shutil.copyfile(os.path.join(build_dir, _FRAMEWORK_METADATA_FILE),
152                             os.path.join(self.extensions[0].cmake_lists_dir, _FRAMEWORK_METADATA_FILE))
153             # Remove unfound frameworks, otherwise develop mode will fail the install
154             self.extensions = [x for x in self.extensions if os.path.exists(self.get_ext_fullpath(x.name))]
155 
156 
157 # python packages required to use horovod in general
158 require_list = ['cloudpickle', 'psutil', 'pyyaml', 'dataclasses;python_version<"3.7"', 'packaging']
159 
160 # framework dependencies
161 tensorflow_require_list = ['tensorflow']
162 tensorflow_cpu_require_list = ['tensorflow-cpu']
163 tensorflow_gpu_require_list = ['tensorflow-gpu']
164 keras_require_list = ['keras>=2.0.8,!=2.0.9,!=2.1.0,!=2.1.1']
165 # pytorch-lightning 1.3.8 is a stable version to work with horovod
166 pytorch_require_list = ['torch']
167 mxnet_require_list = ['mxnet>=1.4.1']
168 pyspark_require_list = ['pyspark>=2.3.2;python_version<"3.8"',
169                         'pyspark>=3.0.0;python_version>="3.8"']
170 spark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0', 'fsspec>=2021.07.0']
171 # https://github.com/ray-project/ray/pull/17465
172 # google-api-core>=2.9.0 depends on protobuf<5.0.0dev,>=3.20.1, which conflicts with
173 #   tensorflow protobuf~=3.20 and pytorch-lightning protobuf<3.20,>=3.9.2
174 ray_require_list = ['ray', 'aioredis<2', 'google-api-core<2.9.0']
175 pytorch_spark_require_list = pytorch_require_list + \
176                              spark_require_list + \
177                              pyspark_require_list + \
178                              ['pytorch_lightning>=1.3.8,<1.5.10']
179 
180 # all frameworks' dependencies
181 all_frameworks_require_list = tensorflow_require_list + \
182                               keras_require_list + \
183                               pytorch_require_list + \
184                               mxnet_require_list + \
185                               spark_require_list + \
186                               pyspark_require_list
187 
188 # python packages required / recommended to develop horovod
189 # these are the earliest versions to work with Python 3.8
190 # keep in sync with Dockerfile.test.cpu
191 # NOTE: do not use versions with +cpu or +gpu here as users would need to add --find-links to pip
192 dev_require_list = ['tensorflow-cpu==2.2.0',
193                     'keras==2.3.1',
194                     'torch==1.4.0',
195                     'torchvision==0.5.0',
196                     'pytorch_lightning>=1.3.8,<1.5.10',
197                     'mxnet==1.5.0',
198                     'pyspark==3.0.1'] + spark_require_list
199 # torchvision 0.5.0 depends on torch==1.4.0
200 
201 # python packages required only to run tests
202 test_require_list = ['mock', 'pytest', 'pytest-forked', 'pytest-subtests', 'parameterized']
203 
204 # Skip cffi if pytorch extension explicitly disabled
205 if not os.environ.get('HOROVOD_WITHOUT_PYTORCH'):
206     require_list.append('cffi>=1.4.0')
207 
208 
209 def get_package_version():
210     return __version__ + "+" + os.environ['HOROVOD_LOCAL_VERSION'] if 'HOROVOD_LOCAL_VERSION' in os.environ else __version__
211 
212 
213 setup(name='horovod',
214       version=get_package_version(),
215       packages=find_packages(),
216       description='Distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.',
217       author='The Horovod Authors',
218       license='Apache 2.0',
219       long_description=textwrap.dedent('''\
220           Horovod is a distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.
221           The goal of Horovod is to make distributed Deep Learning fast and easy to use.'''),
222       url='https://github.com/horovod/horovod',
223       keywords=['deep learning', 'tensorflow', 'keras', 'pytorch', 'mxnet', 'spark', 'AI'],
224       classifiers=[
225           'License :: OSI Approved :: Apache Software License',
226           'Development Status :: 4 - Beta',
227           'Intended Audience :: Developers',
228           'Topic :: Scientific/Engineering :: Artificial Intelligence',
229       ],
230       ext_modules=[tensorflow_mpi_lib, torch_mpi_lib_v2, mxnet_mpi_lib],
231       cmdclass={'build_ext': custom_build_ext},
232       # cffi is required for PyTorch
233       # If cffi is specified in setup_requires, it will need libffi to be installed on the machine,
234       # which is undesirable.  Luckily, `install` action will install cffi before executing build,
235       # so it's only necessary for `build*` or `bdist*` actions.
236       setup_requires=require_list if is_build_action() else [],
237       install_requires=require_list,
238       tests_require=test_require_list,
239       extras_require={
240           'all-frameworks': all_frameworks_require_list,
241           'tensorflow': tensorflow_require_list,
242           'tensorflow-cpu': tensorflow_cpu_require_list,
243           'tensorflow-gpu': tensorflow_gpu_require_list,
244           'keras': keras_require_list,
245           'pytorch': pytorch_require_list,
246           'mxnet': mxnet_require_list,
247           'spark': spark_require_list + pyspark_require_list,
248           'pytorch-spark': pytorch_spark_require_list,
249           'ray': ray_require_list,
250           'dev': dev_require_list,
251           'test': test_require_list,
252       },
253       python_requires='>=3.6',
254       zip_safe=False,
255       entry_points={
256           'console_scripts': [
257               'horovodrun = horovod.runner.launch:run_commandline'
258           ]
259       })
260
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -167,7 +167,7 @@
 mxnet_require_list = ['mxnet>=1.4.1']
 pyspark_require_list = ['pyspark>=2.3.2;python_version<"3.8"',
                         'pyspark>=3.0.0;python_version>="3.8"']
-spark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0', 'fsspec>=2021.07.0']
+spark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0,<11.0', 'fsspec>=2021.07.0']
 # https://github.com/ray-project/ray/pull/17465
 # google-api-core>=2.9.0 depends on protobuf<5.0.0dev,>=3.20.1, which conflicts with
 #   tensorflow protobuf~=3.20 and pytorch-lightning protobuf<3.20,>=3.9.2

{"golden_diff": "diff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -167,7 +167,7 @@\n mxnet_require_list = ['mxnet>=1.4.1']\n pyspark_require_list = ['pyspark>=2.3.2;python_version<\"3.8\"',\n                         'pyspark>=3.0.0;python_version>=\"3.8\"']\n-spark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0', 'fsspec>=2021.07.0']\n+spark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0,<11.0', 'fsspec>=2021.07.0']\n # https://github.com/ray-project/ray/pull/17465\n # google-api-core>=2.9.0 depends on protobuf<5.0.0dev,>=3.20.1, which conflicts with\n #   tensorflow protobuf~=3.20 and pytorch-lightning protobuf<3.20,>=3.9.2\n", "issue": "Spark tests fail with AttributeError: 'pyarrow.lib.Schema' object has no attribute 'to_arrow_schema'\nSpark tests fail with\r\n\r\n```\r\nTraceback (most recent call last):\r\n  File \"/horovod/examples/spark/pytorch/pytorch_spark_mnist.py\", line 122, in <module>\r\n    torch_model = torch_estimator.fit(train_df).setOutputCols(['label_prob'])\r\n  File \"/usr/local/lib/python3.8/dist-packages/horovod/spark/common/estimator.py\", line 35, in fit\r\n    return super(HorovodEstimator, self).fit(df, params)\r\n  File \"/usr/local/lib/python3.8/dist-packages/pyspark/ml/base.py\", line 205, in fit\r\n    return self._fit(dataset)\r\n  File \"/usr/local/lib/python3.8/dist-packages/horovod/spark/common/estimator.py\", line 68, in _fit\r\n    with util.prepare_data(backend.num_processes(),\r\n  File \"/usr/lib/python3.8/contextlib.py\", line 113, in __enter__\r\n    return next(self.gen)\r\n  File \"/usr/local/lib/python3.8/dist-packages/horovod/spark/common/util.py\", line 735, in prepare_data\r\n    dataset_idx = _get_or_create_dataset(key, store, df, feature_columns, label_columns,\r\n  File \"/usr/local/lib/python3.8/dist-packages/horovod/spark/common/util.py\", line 672, in _get_or_create_dataset\r\n    train_rows, val_rows, pq_metadata, avg_row_size = get_simple_meta_from_parquet(\r\n  File \"/usr/local/lib/python3.8/dist-packages/horovod/spark/common/util.py\", line 495, in get_simple_meta_from_parquet\r\n    train_data_schema = train_data.schema.to_arrow_schema()\r\nAttributeError: 'pyarrow.lib.Schema' object has no attribute 'to_arrow_schema'\r\n```\r\nhttps://github.com/horovod/horovod/actions/runs/4025636527/jobs/6919132641\r\n\r\nIt's probably related to the 11.0 release of pyarrow. https://pypi.org/project/pyarrow/#history\n", "before_files": [{"content": "# Copyright 2019 Uber Technologies, Inc. All Rights Reserved.\n# Modifications copyright Microsoft\n# Modifications copyright (C) 2020, NVIDIA CORPORATION. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\nimport atexit\nimport io\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\nimport tempfile\nimport textwrap\n\nfrom setuptools import setup, Extension, find_packages\nfrom setuptools.command.build_ext import build_ext\n\nfrom horovod import __version__\n\n_FRAMEWORK_METADATA_FILE = 'horovod/metadata.json'\n\nclass CMakeExtension(Extension):\n    def __init__(self, name, cmake_lists_dir='.', sources=None, **kwa):\n        if sources is None:\n            sources = []\n        Extension.__init__(self, name, sources=sources, **kwa)\n        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)\n\n\ntensorflow_mpi_lib = CMakeExtension('horovod.tensorflow.mpi_lib',\n                                     cmake_lists_dir='.', sources=[])\ntorch_mpi_lib_v2 = CMakeExtension('horovod.torch.mpi_lib_v2',\n                                     cmake_lists_dir='.', sources=[])\nmxnet_mpi_lib = CMakeExtension('horovod.mxnet.mpi_lib',\n                                     cmake_lists_dir='.', sources=[])\n\ndef is_build_action():\n    if len(sys.argv) <= 1:\n        return False\n\n    if sys.argv[1].startswith('build'):\n        return True\n\n    if sys.argv[1].startswith('bdist'):\n        return True\n\n    if sys.argv[1].startswith('install'):\n        return True\n\n    if sys.argv[1].startswith('develop'):\n        return True\n\ndef get_cmake_bin():\n    from packaging import version\n\n    if 'HOROVOD_CMAKE' in os.environ:\n        return os.environ['HOROVOD_CMAKE']\n\n    cmake_bin = 'cmake'\n    try:\n        out = subprocess.check_output([cmake_bin, '--version'])\n    except OSError:\n        cmake_installed_version = version.parse(\"0.0\")\n    else:\n        cmake_installed_version = version.parse(re.search(r'version\\s*([\\d.]+)', out.decode()).group(1))\n\n    if cmake_installed_version < version.parse(\"3.13.0\"):\n        print(\"Could not find a recent CMake to build Horovod. \"\n              \"Attempting to install CMake 3.13 to a temporary location via pip.\", flush=True)\n        cmake_temp_dir = tempfile.TemporaryDirectory(prefix=\"horovod-cmake-tmp\")\n        atexit.register(cmake_temp_dir.cleanup)\n        try:\n            _ = subprocess.check_output([\"pip\", \"install\", \"--target\", cmake_temp_dir.name, \"cmake~=3.13.0\"])\n        except Exception:\n            raise RuntimeError(\"Failed to install temporary CMake. \"\n                               \"Please update your CMake to 3.13+ or set HOROVOD_CMAKE appropriately.\")\n        cmake_bin = os.path.join(cmake_temp_dir.name, \"bin\", \"run_cmake\")\n        with io.open(cmake_bin, \"w\") as f_run_cmake:\n            f_run_cmake.write(\n                f\"#!/bin/sh\\nPYTHONPATH={cmake_temp_dir.name} {os.path.join(cmake_temp_dir.name, 'bin', 'cmake')} \\\"$@\\\"\")\n        os.chmod(cmake_bin, 0o755)\n\n    return cmake_bin\n\n\nclass custom_build_ext(build_ext):\n    def build_extensions(self):\n        if os.getenv('HOROVOD_SKIP_COMPILE') == '1':\n            # Skip building extensions using CMake\n            print(\"Horovod is being installed without native libraries\")\n            return\n\n        cmake_bin = get_cmake_bin()\n\n        config = 'Debug' if self.debug or os.environ.get('HOROVOD_DEBUG') == \"1\" else 'RelWithDebInfo'\n\n        ext_name = self.extensions[0].name\n        build_dir = self.get_ext_fullpath(ext_name).replace(self.get_ext_filename(ext_name), '')\n        build_dir = os.path.abspath(build_dir)\n\n        cmake_args = ['-DCMAKE_BUILD_TYPE=' + config,\n                      '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(config.upper(), build_dir),\n                      '-DPYTHON_EXECUTABLE:FILEPATH=' + sys.executable]\n\n        make_args = ['-j8'] if not os.environ.get('MAKEFLAGS') else []\n        if self.verbose:\n            make_args.append('VERBOSE=1')\n\n        cmake_build_args = ['--config', config]\n        if make_args:\n            # -- specifies that these args are going to the native build tool: make\n            cmake_build_args += ['--'] + make_args\n\n        cmake_build_dir = os.path.join(self.build_temp, config)\n        if not os.path.exists(cmake_build_dir):\n            os.makedirs(cmake_build_dir)\n\n        config_and_build_commands = [\n            [cmake_bin, self.extensions[0].cmake_lists_dir] + cmake_args,\n            [cmake_bin, '--build', '.'] + cmake_build_args\n        ]\n\n        if self.verbose:\n            print(f\"Running CMake in {cmake_build_dir}:\")\n            for command in config_and_build_commands:\n                print(\" \".join(command))\n            sys.stdout.flush()\n\n        # Config and build the extension\n        try:\n            for command in config_and_build_commands:\n                subprocess.check_call(command, cwd=cmake_build_dir)\n        except OSError as e:\n            raise RuntimeError('CMake failed: {}'.format(str(e)))\n\n        if sys.argv[1].startswith('develop'):\n            # Copy over metadata.json file from build directory\n            shutil.copyfile(os.path.join(build_dir, _FRAMEWORK_METADATA_FILE),\n                            os.path.join(self.extensions[0].cmake_lists_dir, _FRAMEWORK_METADATA_FILE))\n            # Remove unfound frameworks, otherwise develop mode will fail the install\n            self.extensions = [x for x in self.extensions if os.path.exists(self.get_ext_fullpath(x.name))]\n\n\n# python packages required to use horovod in general\nrequire_list = ['cloudpickle', 'psutil', 'pyyaml', 'dataclasses;python_version<\"3.7\"', 'packaging']\n\n# framework dependencies\ntensorflow_require_list = ['tensorflow']\ntensorflow_cpu_require_list = ['tensorflow-cpu']\ntensorflow_gpu_require_list = ['tensorflow-gpu']\nkeras_require_list = ['keras>=2.0.8,!=2.0.9,!=2.1.0,!=2.1.1']\n# pytorch-lightning 1.3.8 is a stable version to work with horovod\npytorch_require_list = ['torch']\nmxnet_require_list = ['mxnet>=1.4.1']\npyspark_require_list = ['pyspark>=2.3.2;python_version<\"3.8\"',\n                        'pyspark>=3.0.0;python_version>=\"3.8\"']\nspark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0', 'fsspec>=2021.07.0']\n# https://github.com/ray-project/ray/pull/17465\n# google-api-core>=2.9.0 depends on protobuf<5.0.0dev,>=3.20.1, which conflicts with\n#   tensorflow protobuf~=3.20 and pytorch-lightning protobuf<3.20,>=3.9.2\nray_require_list = ['ray', 'aioredis<2', 'google-api-core<2.9.0']\npytorch_spark_require_list = pytorch_require_list + \\\n                             spark_require_list + \\\n                             pyspark_require_list + \\\n                             ['pytorch_lightning>=1.3.8,<1.5.10']\n\n# all frameworks' dependencies\nall_frameworks_require_list = tensorflow_require_list + \\\n                              keras_require_list + \\\n                              pytorch_require_list + \\\n                              mxnet_require_list + \\\n                              spark_require_list + \\\n                              pyspark_require_list\n\n# python packages required / recommended to develop horovod\n# these are the earliest versions to work with Python 3.8\n# keep in sync with Dockerfile.test.cpu\n# NOTE: do not use versions with +cpu or +gpu here as users would need to add --find-links to pip\ndev_require_list = ['tensorflow-cpu==2.2.0',\n                    'keras==2.3.1',\n                    'torch==1.4.0',\n                    'torchvision==0.5.0',\n                    'pytorch_lightning>=1.3.8,<1.5.10',\n                    'mxnet==1.5.0',\n                    'pyspark==3.0.1'] + spark_require_list\n# torchvision 0.5.0 depends on torch==1.4.0\n\n# python packages required only to run tests\ntest_require_list = ['mock', 'pytest', 'pytest-forked', 'pytest-subtests', 'parameterized']\n\n# Skip cffi if pytorch extension explicitly disabled\nif not os.environ.get('HOROVOD_WITHOUT_PYTORCH'):\n    require_list.append('cffi>=1.4.0')\n\n\ndef get_package_version():\n    return __version__ + \"+\" + os.environ['HOROVOD_LOCAL_VERSION'] if 'HOROVOD_LOCAL_VERSION' in os.environ else __version__\n\n\nsetup(name='horovod',\n      version=get_package_version(),\n      packages=find_packages(),\n      description='Distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.',\n      author='The Horovod Authors',\n      license='Apache 2.0',\n      long_description=textwrap.dedent('''\\\n          Horovod is a distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.\n          The goal of Horovod is to make distributed Deep Learning fast and easy to use.'''),\n      url='https://github.com/horovod/horovod',\n      keywords=['deep learning', 'tensorflow', 'keras', 'pytorch', 'mxnet', 'spark', 'AI'],\n      classifiers=[\n          'License :: OSI Approved :: Apache Software License',\n          'Development Status :: 4 - Beta',\n          'Intended Audience :: Developers',\n          'Topic :: Scientific/Engineering :: Artificial Intelligence',\n      ],\n      ext_modules=[tensorflow_mpi_lib, torch_mpi_lib_v2, mxnet_mpi_lib],\n      cmdclass={'build_ext': custom_build_ext},\n      # cffi is required for PyTorch\n      # If cffi is specified in setup_requires, it will need libffi to be installed on the machine,\n      # which is undesirable.  Luckily, `install` action will install cffi before executing build,\n      # so it's only necessary for `build*` or `bdist*` actions.\n      setup_requires=require_list if is_build_action() else [],\n      install_requires=require_list,\n      tests_require=test_require_list,\n      extras_require={\n          'all-frameworks': all_frameworks_require_list,\n          'tensorflow': tensorflow_require_list,\n          'tensorflow-cpu': tensorflow_cpu_require_list,\n          'tensorflow-gpu': tensorflow_gpu_require_list,\n          'keras': keras_require_list,\n          'pytorch': pytorch_require_list,\n          'mxnet': mxnet_require_list,\n          'spark': spark_require_list + pyspark_require_list,\n          'pytorch-spark': pytorch_spark_require_list,\n          'ray': ray_require_list,\n          'dev': dev_require_list,\n          'test': test_require_list,\n      },\n      python_requires='>=3.6',\n      zip_safe=False,\n      entry_points={\n          'console_scripts': [\n              'horovodrun = horovod.runner.launch:run_commandline'\n          ]\n      })\n", "path": "setup.py"}], "after_files": [{"content": "# Copyright 2019 Uber Technologies, Inc. All Rights Reserved.\n# Modifications copyright Microsoft\n# Modifications copyright (C) 2020, NVIDIA CORPORATION. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\nimport atexit\nimport io\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\nimport tempfile\nimport textwrap\n\nfrom setuptools import setup, Extension, find_packages\nfrom setuptools.command.build_ext import build_ext\n\nfrom horovod import __version__\n\n_FRAMEWORK_METADATA_FILE = 'horovod/metadata.json'\n\nclass CMakeExtension(Extension):\n    def __init__(self, name, cmake_lists_dir='.', sources=None, **kwa):\n        if sources is None:\n            sources = []\n        Extension.__init__(self, name, sources=sources, **kwa)\n        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)\n\n\ntensorflow_mpi_lib = CMakeExtension('horovod.tensorflow.mpi_lib',\n                                     cmake_lists_dir='.', sources=[])\ntorch_mpi_lib_v2 = CMakeExtension('horovod.torch.mpi_lib_v2',\n                                     cmake_lists_dir='.', sources=[])\nmxnet_mpi_lib = CMakeExtension('horovod.mxnet.mpi_lib',\n                                     cmake_lists_dir='.', sources=[])\n\ndef is_build_action():\n    if len(sys.argv) <= 1:\n        return False\n\n    if sys.argv[1].startswith('build'):\n        return True\n\n    if sys.argv[1].startswith('bdist'):\n        return True\n\n    if sys.argv[1].startswith('install'):\n        return True\n\n    if sys.argv[1].startswith('develop'):\n        return True\n\ndef get_cmake_bin():\n    from packaging import version\n\n    if 'HOROVOD_CMAKE' in os.environ:\n        return os.environ['HOROVOD_CMAKE']\n\n    cmake_bin = 'cmake'\n    try:\n        out = subprocess.check_output([cmake_bin, '--version'])\n    except OSError:\n        cmake_installed_version = version.parse(\"0.0\")\n    else:\n        cmake_installed_version = version.parse(re.search(r'version\\s*([\\d.]+)', out.decode()).group(1))\n\n    if cmake_installed_version < version.parse(\"3.13.0\"):\n        print(\"Could not find a recent CMake to build Horovod. \"\n              \"Attempting to install CMake 3.13 to a temporary location via pip.\", flush=True)\n        cmake_temp_dir = tempfile.TemporaryDirectory(prefix=\"horovod-cmake-tmp\")\n        atexit.register(cmake_temp_dir.cleanup)\n        try:\n            _ = subprocess.check_output([\"pip\", \"install\", \"--target\", cmake_temp_dir.name, \"cmake~=3.13.0\"])\n        except Exception:\n            raise RuntimeError(\"Failed to install temporary CMake. \"\n                               \"Please update your CMake to 3.13+ or set HOROVOD_CMAKE appropriately.\")\n        cmake_bin = os.path.join(cmake_temp_dir.name, \"bin\", \"run_cmake\")\n        with io.open(cmake_bin, \"w\") as f_run_cmake:\n            f_run_cmake.write(\n                f\"#!/bin/sh\\nPYTHONPATH={cmake_temp_dir.name} {os.path.join(cmake_temp_dir.name, 'bin', 'cmake')} \\\"$@\\\"\")\n        os.chmod(cmake_bin, 0o755)\n\n    return cmake_bin\n\n\nclass custom_build_ext(build_ext):\n    def build_extensions(self):\n        if os.getenv('HOROVOD_SKIP_COMPILE') == '1':\n            # Skip building extensions using CMake\n            print(\"Horovod is being installed without native libraries\")\n            return\n\n        cmake_bin = get_cmake_bin()\n\n        config = 'Debug' if self.debug or os.environ.get('HOROVOD_DEBUG') == \"1\" else 'RelWithDebInfo'\n\n        ext_name = self.extensions[0].name\n        build_dir = self.get_ext_fullpath(ext_name).replace(self.get_ext_filename(ext_name), '')\n        build_dir = os.path.abspath(build_dir)\n\n        cmake_args = ['-DCMAKE_BUILD_TYPE=' + config,\n                      '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(config.upper(), build_dir),\n                      '-DPYTHON_EXECUTABLE:FILEPATH=' + sys.executable]\n\n        make_args = ['-j8'] if not os.environ.get('MAKEFLAGS') else []\n        if self.verbose:\n            make_args.append('VERBOSE=1')\n\n        cmake_build_args = ['--config', config]\n        if make_args:\n            # -- specifies that these args are going to the native build tool: make\n            cmake_build_args += ['--'] + make_args\n\n        cmake_build_dir = os.path.join(self.build_temp, config)\n        if not os.path.exists(cmake_build_dir):\n            os.makedirs(cmake_build_dir)\n\n        config_and_build_commands = [\n            [cmake_bin, self.extensions[0].cmake_lists_dir] + cmake_args,\n            [cmake_bin, '--build', '.'] + cmake_build_args\n        ]\n\n        if self.verbose:\n            print(f\"Running CMake in {cmake_build_dir}:\")\n            for command in config_and_build_commands:\n                print(\" \".join(command))\n            sys.stdout.flush()\n\n        # Config and build the extension\n        try:\n            for command in config_and_build_commands:\n                subprocess.check_call(command, cwd=cmake_build_dir)\n        except OSError as e:\n            raise RuntimeError('CMake failed: {}'.format(str(e)))\n\n        if sys.argv[1].startswith('develop'):\n            # Copy over metadata.json file from build directory\n            shutil.copyfile(os.path.join(build_dir, _FRAMEWORK_METADATA_FILE),\n                            os.path.join(self.extensions[0].cmake_lists_dir, _FRAMEWORK_METADATA_FILE))\n            # Remove unfound frameworks, otherwise develop mode will fail the install\n            self.extensions = [x for x in self.extensions if os.path.exists(self.get_ext_fullpath(x.name))]\n\n\n# python packages required to use horovod in general\nrequire_list = ['cloudpickle', 'psutil', 'pyyaml', 'dataclasses;python_version<\"3.7\"', 'packaging']\n\n# framework dependencies\ntensorflow_require_list = ['tensorflow']\ntensorflow_cpu_require_list = ['tensorflow-cpu']\ntensorflow_gpu_require_list = ['tensorflow-gpu']\nkeras_require_list = ['keras>=2.0.8,!=2.0.9,!=2.1.0,!=2.1.1']\n# pytorch-lightning 1.3.8 is a stable version to work with horovod\npytorch_require_list = ['torch']\nmxnet_require_list = ['mxnet>=1.4.1']\npyspark_require_list = ['pyspark>=2.3.2;python_version<\"3.8\"',\n                        'pyspark>=3.0.0;python_version>=\"3.8\"']\nspark_require_list = ['numpy', 'petastorm>=0.12.0', 'pyarrow>=0.15.0,<11.0', 'fsspec>=2021.07.0']\n# https://github.com/ray-project/ray/pull/17465\n# google-api-core>=2.9.0 depends on protobuf<5.0.0dev,>=3.20.1, which conflicts with\n#   tensorflow protobuf~=3.20 and pytorch-lightning protobuf<3.20,>=3.9.2\nray_require_list = ['ray', 'aioredis<2', 'google-api-core<2.9.0']\npytorch_spark_require_list = pytorch_require_list + \\\n                             spark_require_list + \\\n                             pyspark_require_list + \\\n                             ['pytorch_lightning>=1.3.8,<1.5.10']\n\n# all frameworks' dependencies\nall_frameworks_require_list = tensorflow_require_list + \\\n                              keras_require_list + \\\n                              pytorch_require_list + \\\n                              mxnet_require_list + \\\n                              spark_require_list + \\\n                              pyspark_require_list\n\n# python packages required / recommended to develop horovod\n# these are the earliest versions to work with Python 3.8\n# keep in sync with Dockerfile.test.cpu\n# NOTE: do not use versions with +cpu or +gpu here as users would need to add --find-links to pip\ndev_require_list = ['tensorflow-cpu==2.2.0',\n                    'keras==2.3.1',\n                    'torch==1.4.0',\n                    'torchvision==0.5.0',\n                    'pytorch_lightning>=1.3.8,<1.5.10',\n                    'mxnet==1.5.0',\n                    'pyspark==3.0.1'] + spark_require_list\n# torchvision 0.5.0 depends on torch==1.4.0\n\n# python packages required only to run tests\ntest_require_list = ['mock', 'pytest', 'pytest-forked', 'pytest-subtests', 'parameterized']\n\n# Skip cffi if pytorch extension explicitly disabled\nif not os.environ.get('HOROVOD_WITHOUT_PYTORCH'):\n    require_list.append('cffi>=1.4.0')\n\n\ndef get_package_version():\n    return __version__ + \"+\" + os.environ['HOROVOD_LOCAL_VERSION'] if 'HOROVOD_LOCAL_VERSION' in os.environ else __version__\n\n\nsetup(name='horovod',\n      version=get_package_version(),\n      packages=find_packages(),\n      description='Distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.',\n      author='The Horovod Authors',\n      license='Apache 2.0',\n      long_description=textwrap.dedent('''\\\n          Horovod is a distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.\n          The goal of Horovod is to make distributed Deep Learning fast and easy to use.'''),\n      url='https://github.com/horovod/horovod',\n      keywords=['deep learning', 'tensorflow', 'keras', 'pytorch', 'mxnet', 'spark', 'AI'],\n      classifiers=[\n          'License :: OSI Approved :: Apache Software License',\n          'Development Status :: 4 - Beta',\n          'Intended Audience :: Developers',\n          'Topic :: Scientific/Engineering :: Artificial Intelligence',\n      ],\n      ext_modules=[tensorflow_mpi_lib, torch_mpi_lib_v2, mxnet_mpi_lib],\n      cmdclass={'build_ext': custom_build_ext},\n      # cffi is required for PyTorch\n      # If cffi is specified in setup_requires, it will need libffi to be installed on the machine,\n      # which is undesirable.  Luckily, `install` action will install cffi before executing build,\n      # so it's only necessary for `build*` or `bdist*` actions.\n      setup_requires=require_list if is_build_action() else [],\n      install_requires=require_list,\n      tests_require=test_require_list,\n      extras_require={\n          'all-frameworks': all_frameworks_require_list,\n          'tensorflow': tensorflow_require_list,\n          'tensorflow-cpu': tensorflow_cpu_require_list,\n          'tensorflow-gpu': tensorflow_gpu_require_list,\n          'keras': keras_require_list,\n          'pytorch': pytorch_require_list,\n          'mxnet': mxnet_require_list,\n          'spark': spark_require_list + pyspark_require_list,\n          'pytorch-spark': pytorch_spark_require_list,\n          'ray': ray_require_list,\n          'dev': dev_require_list,\n          'test': test_require_list,\n      },\n      python_requires='>=3.6',\n      zip_safe=False,\n      entry_points={\n          'console_scripts': [\n              'horovodrun = horovod.runner.launch:run_commandline'\n          ]\n      })\n", "path": "setup.py"}]}