We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
TensorFlow examples do not work with TensorFlow==2.0.0.
**Conditions**
- Optuna version: 0.16.0
- Python version: 3.7.2
- OS: macOS 10.13.6
- Machine Learning library to be optimized: TensorFlow 2.0.0

**Code to reproduce**

`examples/tensorflow_*` and `examples/pruning/tensorflow_*` do not work with TensorFlow==2.0.0.

```
$ python examples/tensorflow_estimator_simple.py

Traceback (most recent call last):
  File "examples/tensorflow_estimator_simple.py", line 137, in <module>
    tf.app.run()
AttributeError: module 'tensorflow' has no attribute 'app'
```

```
$ python examples/tensorflow_eager_simple.py
Traceback (most recent call last):
  File "examples/tensorflow_eager_simple.py", line 22, in <module>
    import tensorflow.contrib.eager as tfe
ModuleNotFoundError: No module named 'tensorflow.contrib'
```

Related PR #564.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `setup.py`
Content:
```
1 import os
2 import sys
3 
4 import pkg_resources
5 from setuptools import find_packages
6 from setuptools import setup
7 
8 from typing import Dict  # NOQA
9 from typing import List  # NOQA
10 from typing import Optional  # NOQA
11 
12 
13 def get_version():
14     # type: () -> str
15 
16     version_filepath = os.path.join(os.path.dirname(__file__), 'optuna', 'version.py')
17     with open(version_filepath) as f:
18         for line in f:
19             if line.startswith('__version__'):
20                 return line.strip().split()[-1][1:-1]
21     assert False
22 
23 
24 def get_long_description():
25     # type: () -> str
26 
27     readme_filepath = os.path.join(os.path.dirname(__file__), 'README.md')
28     with open(readme_filepath) as f:
29         return f.read()
30 
31 
32 def get_install_requires():
33     # type: () -> List[str]
34 
35     return [
36         'alembic',
37         'cliff',
38         'colorlog',
39         'numpy',
40         'scipy!=1.4.0',
41         'sqlalchemy>=1.1.0',
42         'tqdm',
43         'joblib',
44     ]
45 
46 
47 def get_tests_require():
48     # type: () -> List[str]
49 
50     return get_extras_require()['testing']
51 
52 
53 def get_extras_require():
54     # type: () -> Dict[str, List[str]]
55 
56     requirements = {
57         'checking': [
58             'autopep8',
59             'hacking',
60             'mypy',
61         ],
62         'codecov': [
63             'codecov',
64             'pytest-cov',
65         ],
66         'doctest': [
67             'pandas',
68             # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.
69             # See https://github.com/optuna/optuna/issues/825 for further details.
70             'scikit-learn>=0.19.0,<=0.22.0',
71         ],
72         'document': [
73             'lightgbm',
74             'sphinx',
75             'sphinx_rtd_theme',
76         ],
77         'example': [
78             'catboost',
79             'chainer',
80             'lightgbm',
81             'mlflow',
82             'mxnet',
83             'scikit-image',
84             # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.
85             # See https://github.com/optuna/optuna/issues/825 for further details.
86             'scikit-learn<=0.22.0',
87             'xgboost',
88         ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])
89         + ([
90             'dask[dataframe]',
91             'dask-ml',
92             'keras',
93             'pytorch-ignite',
94             'pytorch-lightning',
95             # TODO(Yanase): Update examples to support TensorFlow 2.0.
96             # See https://github.com/optuna/optuna/issues/565 for further details.
97             'tensorflow<2.0.0',
98             'torch',
99             'torchvision>=0.5.0'
100         ] if sys.version_info[:2] < (3, 8) else []),
101         'testing': [
102             'bokeh',
103             'chainer>=5.0.0',
104             'cma',
105             'lightgbm',
106             'mock',
107             'mpi4py',
108             'mxnet',
109             'pandas',
110             'plotly>=4.0.0',
111             'pytest',
112             # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.
113             # See https://github.com/optuna/optuna/issues/825 for further details.
114             'scikit-learn>=0.19.0,<=0.22.0',
115             'scikit-optimize',
116             'xgboost',
117         ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])
118         + ([
119             'keras',
120             'pytorch-ignite',
121             'pytorch-lightning',
122             'tensorflow',
123             'tensorflow-datasets',
124             'torch',
125             'torchvision>=0.5.0'
126         ] if sys.version_info[:2] < (3, 8) else []),
127     }
128 
129     return requirements
130 
131 
132 def find_any_distribution(pkgs):
133     # type: (List[str]) -> Optional[pkg_resources.Distribution]
134 
135     for pkg in pkgs:
136         try:
137             return pkg_resources.get_distribution(pkg)
138         except pkg_resources.DistributionNotFound:
139             pass
140     return None
141 
142 
143 pfnopt_pkg = find_any_distribution(['pfnopt'])
144 if pfnopt_pkg is not None:
145     msg = 'We detected that PFNOpt is installed in your environment.\n' \
146         'PFNOpt has been renamed Optuna. Please uninstall the old\n' \
147         'PFNOpt in advance (e.g. by executing `$ pip uninstall pfnopt`).'
148     print(msg)
149     exit(1)
150 
151 setup(
152     name='optuna',
153     version=get_version(),
154     description='A hyperparameter optimization framework',
155     long_description=get_long_description(),
156     long_description_content_type='text/markdown',
157     author='Takuya Akiba',
158     author_email='akiba@preferred.jp',
159     url='https://optuna.org/',
160     packages=find_packages(),
161     package_data={
162         'optuna': [
163             'storages/rdb/alembic.ini',
164             'storages/rdb/alembic/*.*',
165             'storages/rdb/alembic/versions/*.*'
166         ]
167     },
168     install_requires=get_install_requires(),
169     tests_require=get_tests_require(),
170     extras_require=get_extras_require(),
171     entry_points={'console_scripts': ['optuna = optuna.cli:main']})
172
```

Path: `examples/tensorflow_eager_simple.py`
Content:
```
1 """
2 Optuna example that optimizes multi-layer perceptrons using Tensorflow (Eager Execution).
3 
4 In this example, we optimize the validation accuracy of hand-written digit recognition using
5 Tensorflow and MNIST. We optimize the neural network architecture as well as the optimizer
6 configuration.
7 
8 We have the following two ways to execute this example:
9 
10 (1) Execute this code directly.
11     $ python tensorflow_eager_simple.py
12 
13 
14 (2) Execute through CLI.
15     $ STUDY_NAME=`optuna create-study --direction maximize --storage sqlite:///example.db`
16     $ optuna study optimize tensorflow_eager_simple.py objective --n-trials=100 \
17       --study $STUDY_NAME --storage sqlite:///example.db
18 
19 """
20 
21 import tensorflow as tf
22 from tensorflow.keras.datasets import mnist
23 
24 import optuna
25 
26 N_TRAIN_EXAMPLES = 3000
27 N_TEST_EXAMPLES = 1000
28 BATCHSIZE = 128
29 CLASSES = 10
30 EPOCHS = 1
31 tf.enable_eager_execution()
32 
33 
34 def create_model(trial):
35     # We optimize the numbers of layers, their units and weight decay parameter.
36     n_layers = trial.suggest_int('n_layers', 1, 3)
37     weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)
38     model = tf.keras.Sequential()
39     model.add(tf.keras.layers.Flatten())
40     for i in range(n_layers):
41         num_hidden = int(trial.suggest_loguniform('n_units_l{}'.format(i), 4, 128))
42         model.add(
43             tf.keras.layers.Dense(num_hidden,
44                                   activation='relu',
45                                   kernel_regularizer=tf.keras.regularizers.l2(weight_decay)))
46     model.add(
47         tf.keras.layers.Dense(CLASSES, kernel_regularizer=tf.keras.regularizers.l2(weight_decay)))
48     return model
49 
50 
51 def create_optimizer(trial):
52     # We optimize the choice of optimizers as well as their parameters.
53     kwargs = {}
54     optimizer_options = ['RMSPropOptimizer', 'AdamOptimizer', 'MomentumOptimizer']
55     optimizer_selected = trial.suggest_categorical('optimizer', optimizer_options)
56     if optimizer_selected == 'RMSPropOptimizer':
57         kwargs['learning_rate'] = trial.suggest_loguniform('rmsprop_learning_rate', 1e-5, 1e-1)
58         kwargs['decay'] = trial.suggest_uniform('rmsprop_decay', 0.85, 0.99)
59         kwargs['momentum'] = trial.suggest_loguniform('rmsprop_momentum', 1e-5, 1e-1)
60     elif optimizer_selected == 'AdamOptimizer':
61         kwargs['learning_rate'] = trial.suggest_loguniform('adam_learning_rate', 1e-5, 1e-1)
62     elif optimizer_selected == 'MomentumOptimizer':
63         kwargs['learning_rate'] = trial.suggest_loguniform('momentum_opt_learning_rate', 1e-5,
64                                                            1e-1)
65         kwargs['momentum'] = trial.suggest_loguniform('momentum_opt_momentum', 1e-5, 1e-1)
66 
67     optimizer = getattr(tf.train, optimizer_selected)(**kwargs)
68     return optimizer
69 
70 
71 def learn(model, optimizer, dataset, mode='eval'):
72     accuracy = tf.contrib.eager.metrics.Accuracy('accuracy', dtype=tf.float32)
73 
74     for batch, (images, labels) in enumerate(dataset):
75         with tf.GradientTape() as tape:
76             logits = model(images, training=(mode == 'train'))
77             loss_value = tf.reduce_mean(
78                 tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels))
79             if mode == 'eval':
80                 accuracy(tf.argmax(logits, axis=1, output_type=tf.int64),
81                          tf.cast(labels, tf.int64))
82             else:
83                 grads = tape.gradient(loss_value, model.variables)
84                 optimizer.apply_gradients(zip(grads, model.variables))
85 
86     if mode == 'eval':
87         return accuracy
88 
89 
90 def get_mnist():
91     (x_train, y_train), (x_test, y_test) = mnist.load_data()
92     x_train = x_train.astype('float32') / 255
93     x_test = x_test.astype('float32') / 255
94 
95     y_train = y_train.astype('int32')
96     y_test = y_test.astype('int32')
97 
98     train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train))
99     train_ds = train_ds.shuffle(60000).batch(BATCHSIZE).take(N_TRAIN_EXAMPLES)
100 
101     test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))
102     test_ds = test_ds.shuffle(10000).batch(BATCHSIZE).take(N_TEST_EXAMPLES)
103     return train_ds, test_ds
104 
105 
106 # FYI: Objective functions can take additional arguments
107 # (https://optuna.readthedocs.io/en/stable/faq.html#objective-func-additional-args).
108 def objective(trial):
109     # Get MNIST data.
110     train_ds, test_ds = get_mnist()
111 
112     # Build model and optimizer.
113     model = create_model(trial)
114     optimizer = create_optimizer(trial)
115 
116     # Training and validating cycle.
117     with tf.device("/cpu:0"):
118         for _ in range(EPOCHS):
119             learn(model, optimizer, train_ds, 'train')
120 
121         accuracy = learn(model, optimizer, test_ds, 'eval')
122 
123     # Return last validation accuracy.
124     return accuracy.result()
125 
126 
127 if __name__ == '__main__':
128     study = optuna.create_study(direction='maximize')
129     study.optimize(objective, n_trials=100)
130 
131     print('Number of finished trials: ', len(study.trials))
132 
133     print('Best trial:')
134     trial = study.best_trial
135 
136     print('  Value: ', trial.value)
137 
138     print('  Params: ')
139     for key, value in trial.params.items():
140         print('    {}: {}'.format(key, value))
141
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/tensorflow_eager_simple.py b/examples/tensorflow_eager_simple.py
--- a/examples/tensorflow_eager_simple.py
+++ b/examples/tensorflow_eager_simple.py
@@ -18,17 +18,20 @@
 
 """
 
+import pkg_resources
 import tensorflow as tf
 from tensorflow.keras.datasets import mnist
 
 import optuna
 
+if pkg_resources.parse_version(tf.__version__) < pkg_resources.parse_version('2.0.0'):
+    raise RuntimeError('tensorflow>=2.0.0 is required for this example.')
+
 N_TRAIN_EXAMPLES = 3000
 N_TEST_EXAMPLES = 1000
 BATCHSIZE = 128
 CLASSES = 10
 EPOCHS = 1
-tf.enable_eager_execution()
 
 
 def create_model(trial):
@@ -51,25 +54,25 @@
 def create_optimizer(trial):
     # We optimize the choice of optimizers as well as their parameters.
     kwargs = {}
-    optimizer_options = ['RMSPropOptimizer', 'AdamOptimizer', 'MomentumOptimizer']
+    optimizer_options = ['RMSprop', 'Adam', 'SGD']
     optimizer_selected = trial.suggest_categorical('optimizer', optimizer_options)
-    if optimizer_selected == 'RMSPropOptimizer':
+    if optimizer_selected == 'RMSprop':
         kwargs['learning_rate'] = trial.suggest_loguniform('rmsprop_learning_rate', 1e-5, 1e-1)
         kwargs['decay'] = trial.suggest_uniform('rmsprop_decay', 0.85, 0.99)
         kwargs['momentum'] = trial.suggest_loguniform('rmsprop_momentum', 1e-5, 1e-1)
-    elif optimizer_selected == 'AdamOptimizer':
+    elif optimizer_selected == 'Adam':
         kwargs['learning_rate'] = trial.suggest_loguniform('adam_learning_rate', 1e-5, 1e-1)
-    elif optimizer_selected == 'MomentumOptimizer':
-        kwargs['learning_rate'] = trial.suggest_loguniform('momentum_opt_learning_rate', 1e-5,
+    elif optimizer_selected == 'SGD':
+        kwargs['learning_rate'] = trial.suggest_loguniform('sgd_opt_learning_rate', 1e-5,
                                                            1e-1)
-        kwargs['momentum'] = trial.suggest_loguniform('momentum_opt_momentum', 1e-5, 1e-1)
+        kwargs['momentum'] = trial.suggest_loguniform('sgd_opt_momentum', 1e-5, 1e-1)
 
-    optimizer = getattr(tf.train, optimizer_selected)(**kwargs)
+    optimizer = getattr(tf.optimizers, optimizer_selected)(**kwargs)
     return optimizer
 
 
 def learn(model, optimizer, dataset, mode='eval'):
-    accuracy = tf.contrib.eager.metrics.Accuracy('accuracy', dtype=tf.float32)
+    accuracy = tf.metrics.Accuracy('accuracy', dtype=tf.float32)
 
     for batch, (images, labels) in enumerate(dataset):
         with tf.GradientTape() as tape:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -92,9 +92,7 @@
             'keras',
             'pytorch-ignite',
             'pytorch-lightning',
-            # TODO(Yanase): Update examples to support TensorFlow 2.0.
-            # See https://github.com/optuna/optuna/issues/565 for further details.
-            'tensorflow<2.0.0',
+            'tensorflow>=2.0.0',
             'torch',
             'torchvision>=0.5.0'
         ] if sys.version_info[:2] < (3, 8) else []),

{"golden_diff": "diff --git a/examples/tensorflow_eager_simple.py b/examples/tensorflow_eager_simple.py\n--- a/examples/tensorflow_eager_simple.py\n+++ b/examples/tensorflow_eager_simple.py\n@@ -18,17 +18,20 @@\n \n \"\"\"\n \n+import pkg_resources\n import tensorflow as tf\n from tensorflow.keras.datasets import mnist\n \n import optuna\n \n+if pkg_resources.parse_version(tf.__version__) < pkg_resources.parse_version('2.0.0'):\n+    raise RuntimeError('tensorflow>=2.0.0 is required for this example.')\n+\n N_TRAIN_EXAMPLES = 3000\n N_TEST_EXAMPLES = 1000\n BATCHSIZE = 128\n CLASSES = 10\n EPOCHS = 1\n-tf.enable_eager_execution()\n \n \n def create_model(trial):\n@@ -51,25 +54,25 @@\n def create_optimizer(trial):\n     # We optimize the choice of optimizers as well as their parameters.\n     kwargs = {}\n-    optimizer_options = ['RMSPropOptimizer', 'AdamOptimizer', 'MomentumOptimizer']\n+    optimizer_options = ['RMSprop', 'Adam', 'SGD']\n     optimizer_selected = trial.suggest_categorical('optimizer', optimizer_options)\n-    if optimizer_selected == 'RMSPropOptimizer':\n+    if optimizer_selected == 'RMSprop':\n         kwargs['learning_rate'] = trial.suggest_loguniform('rmsprop_learning_rate', 1e-5, 1e-1)\n         kwargs['decay'] = trial.suggest_uniform('rmsprop_decay', 0.85, 0.99)\n         kwargs['momentum'] = trial.suggest_loguniform('rmsprop_momentum', 1e-5, 1e-1)\n-    elif optimizer_selected == 'AdamOptimizer':\n+    elif optimizer_selected == 'Adam':\n         kwargs['learning_rate'] = trial.suggest_loguniform('adam_learning_rate', 1e-5, 1e-1)\n-    elif optimizer_selected == 'MomentumOptimizer':\n-        kwargs['learning_rate'] = trial.suggest_loguniform('momentum_opt_learning_rate', 1e-5,\n+    elif optimizer_selected == 'SGD':\n+        kwargs['learning_rate'] = trial.suggest_loguniform('sgd_opt_learning_rate', 1e-5,\n                                                            1e-1)\n-        kwargs['momentum'] = trial.suggest_loguniform('momentum_opt_momentum', 1e-5, 1e-1)\n+        kwargs['momentum'] = trial.suggest_loguniform('sgd_opt_momentum', 1e-5, 1e-1)\n \n-    optimizer = getattr(tf.train, optimizer_selected)(**kwargs)\n+    optimizer = getattr(tf.optimizers, optimizer_selected)(**kwargs)\n     return optimizer\n \n \n def learn(model, optimizer, dataset, mode='eval'):\n-    accuracy = tf.contrib.eager.metrics.Accuracy('accuracy', dtype=tf.float32)\n+    accuracy = tf.metrics.Accuracy('accuracy', dtype=tf.float32)\n \n     for batch, (images, labels) in enumerate(dataset):\n         with tf.GradientTape() as tape:\ndiff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -92,9 +92,7 @@\n             'keras',\n             'pytorch-ignite',\n             'pytorch-lightning',\n-            # TODO(Yanase): Update examples to support TensorFlow 2.0.\n-            # See https://github.com/optuna/optuna/issues/565 for further details.\n-            'tensorflow<2.0.0',\n+            'tensorflow>=2.0.0',\n             'torch',\n             'torchvision>=0.5.0'\n         ] if sys.version_info[:2] < (3, 8) else []),\n", "issue": "TensorFlow examples do not work with TensorFlow==2.0.0.\n**Conditions**\r\n- Optuna version: 0.16.0\r\n- Python version: 3.7.2\r\n- OS: macOS 10.13.6\r\n- Machine Learning library to be optimized: TensorFlow 2.0.0\r\n\r\n**Code to reproduce**\r\n\r\n`examples/tensorflow_*` and `examples/pruning/tensorflow_*` do not work with TensorFlow==2.0.0.\r\n\r\n```\r\n$ python examples/tensorflow_estimator_simple.py\r\n\r\nTraceback (most recent call last):\r\n  File \"examples/tensorflow_estimator_simple.py\", line 137, in <module>\r\n    tf.app.run()\r\nAttributeError: module 'tensorflow' has no attribute 'app'\r\n```\r\n\r\n```\r\n$ python examples/tensorflow_eager_simple.py\r\nTraceback (most recent call last):\r\n  File \"examples/tensorflow_eager_simple.py\", line 22, in <module>\r\n    import tensorflow.contrib.eager as tfe\r\nModuleNotFoundError: No module named 'tensorflow.contrib'\r\n```\r\n\r\nRelated PR #564.\n", "before_files": [{"content": "import os\nimport sys\n\nimport pkg_resources\nfrom setuptools import find_packages\nfrom setuptools import setup\n\nfrom typing import Dict  # NOQA\nfrom typing import List  # NOQA\nfrom typing import Optional  # NOQA\n\n\ndef get_version():\n    # type: () -> str\n\n    version_filepath = os.path.join(os.path.dirname(__file__), 'optuna', 'version.py')\n    with open(version_filepath) as f:\n        for line in f:\n            if line.startswith('__version__'):\n                return line.strip().split()[-1][1:-1]\n    assert False\n\n\ndef get_long_description():\n    # type: () -> str\n\n    readme_filepath = os.path.join(os.path.dirname(__file__), 'README.md')\n    with open(readme_filepath) as f:\n        return f.read()\n\n\ndef get_install_requires():\n    # type: () -> List[str]\n\n    return [\n        'alembic',\n        'cliff',\n        'colorlog',\n        'numpy',\n        'scipy!=1.4.0',\n        'sqlalchemy>=1.1.0',\n        'tqdm',\n        'joblib',\n    ]\n\n\ndef get_tests_require():\n    # type: () -> List[str]\n\n    return get_extras_require()['testing']\n\n\ndef get_extras_require():\n    # type: () -> Dict[str, List[str]]\n\n    requirements = {\n        'checking': [\n            'autopep8',\n            'hacking',\n            'mypy',\n        ],\n        'codecov': [\n            'codecov',\n            'pytest-cov',\n        ],\n        'doctest': [\n            'pandas',\n            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.\n            # See https://github.com/optuna/optuna/issues/825 for further details.\n            'scikit-learn>=0.19.0,<=0.22.0',\n        ],\n        'document': [\n            'lightgbm',\n            'sphinx',\n            'sphinx_rtd_theme',\n        ],\n        'example': [\n            'catboost',\n            'chainer',\n            'lightgbm',\n            'mlflow',\n            'mxnet',\n            'scikit-image',\n            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.\n            # See https://github.com/optuna/optuna/issues/825 for further details.\n            'scikit-learn<=0.22.0',\n            'xgboost',\n        ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])\n        + ([\n            'dask[dataframe]',\n            'dask-ml',\n            'keras',\n            'pytorch-ignite',\n            'pytorch-lightning',\n            # TODO(Yanase): Update examples to support TensorFlow 2.0.\n            # See https://github.com/optuna/optuna/issues/565 for further details.\n            'tensorflow<2.0.0',\n            'torch',\n            'torchvision>=0.5.0'\n        ] if sys.version_info[:2] < (3, 8) else []),\n        'testing': [\n            'bokeh',\n            'chainer>=5.0.0',\n            'cma',\n            'lightgbm',\n            'mock',\n            'mpi4py',\n            'mxnet',\n            'pandas',\n            'plotly>=4.0.0',\n            'pytest',\n            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.\n            # See https://github.com/optuna/optuna/issues/825 for further details.\n            'scikit-learn>=0.19.0,<=0.22.0',\n            'scikit-optimize',\n            'xgboost',\n        ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])\n        + ([\n            'keras',\n            'pytorch-ignite',\n            'pytorch-lightning',\n            'tensorflow',\n            'tensorflow-datasets',\n            'torch',\n            'torchvision>=0.5.0'\n        ] if sys.version_info[:2] < (3, 8) else []),\n    }\n\n    return requirements\n\n\ndef find_any_distribution(pkgs):\n    # type: (List[str]) -> Optional[pkg_resources.Distribution]\n\n    for pkg in pkgs:\n        try:\n            return pkg_resources.get_distribution(pkg)\n        except pkg_resources.DistributionNotFound:\n            pass\n    return None\n\n\npfnopt_pkg = find_any_distribution(['pfnopt'])\nif pfnopt_pkg is not None:\n    msg = 'We detected that PFNOpt is installed in your environment.\\n' \\\n        'PFNOpt has been renamed Optuna. Please uninstall the old\\n' \\\n        'PFNOpt in advance (e.g. by executing `$ pip uninstall pfnopt`).'\n    print(msg)\n    exit(1)\n\nsetup(\n    name='optuna',\n    version=get_version(),\n    description='A hyperparameter optimization framework',\n    long_description=get_long_description(),\n    long_description_content_type='text/markdown',\n    author='Takuya Akiba',\n    author_email='akiba@preferred.jp',\n    url='https://optuna.org/',\n    packages=find_packages(),\n    package_data={\n        'optuna': [\n            'storages/rdb/alembic.ini',\n            'storages/rdb/alembic/*.*',\n            'storages/rdb/alembic/versions/*.*'\n        ]\n    },\n    install_requires=get_install_requires(),\n    tests_require=get_tests_require(),\n    extras_require=get_extras_require(),\n    entry_points={'console_scripts': ['optuna = optuna.cli:main']})\n", "path": "setup.py"}, {"content": "\"\"\"\nOptuna example that optimizes multi-layer perceptrons using Tensorflow (Eager Execution).\n\nIn this example, we optimize the validation accuracy of hand-written digit recognition using\nTensorflow and MNIST. We optimize the neural network architecture as well as the optimizer\nconfiguration.\n\nWe have the following two ways to execute this example:\n\n(1) Execute this code directly.\n    $ python tensorflow_eager_simple.py\n\n\n(2) Execute through CLI.\n    $ STUDY_NAME=`optuna create-study --direction maximize --storage sqlite:///example.db`\n    $ optuna study optimize tensorflow_eager_simple.py objective --n-trials=100 \\\n      --study $STUDY_NAME --storage sqlite:///example.db\n\n\"\"\"\n\nimport tensorflow as tf\nfrom tensorflow.keras.datasets import mnist\n\nimport optuna\n\nN_TRAIN_EXAMPLES = 3000\nN_TEST_EXAMPLES = 1000\nBATCHSIZE = 128\nCLASSES = 10\nEPOCHS = 1\ntf.enable_eager_execution()\n\n\ndef create_model(trial):\n    # We optimize the numbers of layers, their units and weight decay parameter.\n    n_layers = trial.suggest_int('n_layers', 1, 3)\n    weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)\n    model = tf.keras.Sequential()\n    model.add(tf.keras.layers.Flatten())\n    for i in range(n_layers):\n        num_hidden = int(trial.suggest_loguniform('n_units_l{}'.format(i), 4, 128))\n        model.add(\n            tf.keras.layers.Dense(num_hidden,\n                                  activation='relu',\n                                  kernel_regularizer=tf.keras.regularizers.l2(weight_decay)))\n    model.add(\n        tf.keras.layers.Dense(CLASSES, kernel_regularizer=tf.keras.regularizers.l2(weight_decay)))\n    return model\n\n\ndef create_optimizer(trial):\n    # We optimize the choice of optimizers as well as their parameters.\n    kwargs = {}\n    optimizer_options = ['RMSPropOptimizer', 'AdamOptimizer', 'MomentumOptimizer']\n    optimizer_selected = trial.suggest_categorical('optimizer', optimizer_options)\n    if optimizer_selected == 'RMSPropOptimizer':\n        kwargs['learning_rate'] = trial.suggest_loguniform('rmsprop_learning_rate', 1e-5, 1e-1)\n        kwargs['decay'] = trial.suggest_uniform('rmsprop_decay', 0.85, 0.99)\n        kwargs['momentum'] = trial.suggest_loguniform('rmsprop_momentum', 1e-5, 1e-1)\n    elif optimizer_selected == 'AdamOptimizer':\n        kwargs['learning_rate'] = trial.suggest_loguniform('adam_learning_rate', 1e-5, 1e-1)\n    elif optimizer_selected == 'MomentumOptimizer':\n        kwargs['learning_rate'] = trial.suggest_loguniform('momentum_opt_learning_rate', 1e-5,\n                                                           1e-1)\n        kwargs['momentum'] = trial.suggest_loguniform('momentum_opt_momentum', 1e-5, 1e-1)\n\n    optimizer = getattr(tf.train, optimizer_selected)(**kwargs)\n    return optimizer\n\n\ndef learn(model, optimizer, dataset, mode='eval'):\n    accuracy = tf.contrib.eager.metrics.Accuracy('accuracy', dtype=tf.float32)\n\n    for batch, (images, labels) in enumerate(dataset):\n        with tf.GradientTape() as tape:\n            logits = model(images, training=(mode == 'train'))\n            loss_value = tf.reduce_mean(\n                tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels))\n            if mode == 'eval':\n                accuracy(tf.argmax(logits, axis=1, output_type=tf.int64),\n                         tf.cast(labels, tf.int64))\n            else:\n                grads = tape.gradient(loss_value, model.variables)\n                optimizer.apply_gradients(zip(grads, model.variables))\n\n    if mode == 'eval':\n        return accuracy\n\n\ndef get_mnist():\n    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n    x_train = x_train.astype('float32') / 255\n    x_test = x_test.astype('float32') / 255\n\n    y_train = y_train.astype('int32')\n    y_test = y_test.astype('int32')\n\n    train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n    train_ds = train_ds.shuffle(60000).batch(BATCHSIZE).take(N_TRAIN_EXAMPLES)\n\n    test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n    test_ds = test_ds.shuffle(10000).batch(BATCHSIZE).take(N_TEST_EXAMPLES)\n    return train_ds, test_ds\n\n\n# FYI: Objective functions can take additional arguments\n# (https://optuna.readthedocs.io/en/stable/faq.html#objective-func-additional-args).\ndef objective(trial):\n    # Get MNIST data.\n    train_ds, test_ds = get_mnist()\n\n    # Build model and optimizer.\n    model = create_model(trial)\n    optimizer = create_optimizer(trial)\n\n    # Training and validating cycle.\n    with tf.device(\"/cpu:0\"):\n        for _ in range(EPOCHS):\n            learn(model, optimizer, train_ds, 'train')\n\n        accuracy = learn(model, optimizer, test_ds, 'eval')\n\n    # Return last validation accuracy.\n    return accuracy.result()\n\n\nif __name__ == '__main__':\n    study = optuna.create_study(direction='maximize')\n    study.optimize(objective, n_trials=100)\n\n    print('Number of finished trials: ', len(study.trials))\n\n    print('Best trial:')\n    trial = study.best_trial\n\n    print('  Value: ', trial.value)\n\n    print('  Params: ')\n    for key, value in trial.params.items():\n        print('    {}: {}'.format(key, value))\n", "path": "examples/tensorflow_eager_simple.py"}], "after_files": [{"content": "import os\nimport sys\n\nimport pkg_resources\nfrom setuptools import find_packages\nfrom setuptools import setup\n\nfrom typing import Dict  # NOQA\nfrom typing import List  # NOQA\nfrom typing import Optional  # NOQA\n\n\ndef get_version():\n    # type: () -> str\n\n    version_filepath = os.path.join(os.path.dirname(__file__), 'optuna', 'version.py')\n    with open(version_filepath) as f:\n        for line in f:\n            if line.startswith('__version__'):\n                return line.strip().split()[-1][1:-1]\n    assert False\n\n\ndef get_long_description():\n    # type: () -> str\n\n    readme_filepath = os.path.join(os.path.dirname(__file__), 'README.md')\n    with open(readme_filepath) as f:\n        return f.read()\n\n\ndef get_install_requires():\n    # type: () -> List[str]\n\n    return [\n        'alembic',\n        'cliff',\n        'colorlog',\n        'numpy',\n        'scipy!=1.4.0',\n        'sqlalchemy>=1.1.0',\n        'tqdm',\n        'joblib',\n    ]\n\n\ndef get_tests_require():\n    # type: () -> List[str]\n\n    return get_extras_require()['testing']\n\n\ndef get_extras_require():\n    # type: () -> Dict[str, List[str]]\n\n    requirements = {\n        'checking': [\n            'autopep8',\n            'hacking',\n            'mypy',\n        ],\n        'codecov': [\n            'codecov',\n            'pytest-cov',\n        ],\n        'doctest': [\n            'pandas',\n            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.\n            # See https://github.com/optuna/optuna/issues/825 for further details.\n            'scikit-learn>=0.19.0,<=0.22.0',\n        ],\n        'document': [\n            'lightgbm',\n            'sphinx',\n            'sphinx_rtd_theme',\n        ],\n        'example': [\n            'catboost',\n            'chainer',\n            'lightgbm',\n            'mlflow',\n            'mxnet',\n            'scikit-image',\n            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.\n            # See https://github.com/optuna/optuna/issues/825 for further details.\n            'scikit-learn<=0.22.0',\n            'xgboost',\n        ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])\n        + ([\n            'dask[dataframe]',\n            'dask-ml',\n            'keras',\n            'pytorch-ignite',\n            'pytorch-lightning',\n            'tensorflow>=2.0.0',\n            'torch',\n            'torchvision>=0.5.0'\n        ] if sys.version_info[:2] < (3, 8) else []),\n        'testing': [\n            'bokeh',\n            'chainer>=5.0.0',\n            'cma',\n            'lightgbm',\n            'mock',\n            'mpi4py',\n            'mxnet',\n            'pandas',\n            'plotly>=4.0.0',\n            'pytest',\n            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.\n            # See https://github.com/optuna/optuna/issues/825 for further details.\n            'scikit-learn>=0.19.0,<=0.22.0',\n            'scikit-optimize',\n            'xgboost',\n        ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])\n        + ([\n            'keras',\n            'pytorch-ignite',\n            'pytorch-lightning',\n            'tensorflow',\n            'tensorflow-datasets',\n            'torch',\n            'torchvision>=0.5.0'\n        ] if sys.version_info[:2] < (3, 8) else []),\n    }\n\n    return requirements\n\n\ndef find_any_distribution(pkgs):\n    # type: (List[str]) -> Optional[pkg_resources.Distribution]\n\n    for pkg in pkgs:\n        try:\n            return pkg_resources.get_distribution(pkg)\n        except pkg_resources.DistributionNotFound:\n            pass\n    return None\n\n\npfnopt_pkg = find_any_distribution(['pfnopt'])\nif pfnopt_pkg is not None:\n    msg = 'We detected that PFNOpt is installed in your environment.\\n' \\\n        'PFNOpt has been renamed Optuna. Please uninstall the old\\n' \\\n        'PFNOpt in advance (e.g. by executing `$ pip uninstall pfnopt`).'\n    print(msg)\n    exit(1)\n\nsetup(\n    name='optuna',\n    version=get_version(),\n    description='A hyperparameter optimization framework',\n    long_description=get_long_description(),\n    long_description_content_type='text/markdown',\n    author='Takuya Akiba',\n    author_email='akiba@preferred.jp',\n    url='https://optuna.org/',\n    packages=find_packages(),\n    package_data={\n        'optuna': [\n            'storages/rdb/alembic.ini',\n            'storages/rdb/alembic/*.*',\n            'storages/rdb/alembic/versions/*.*'\n        ]\n    },\n    install_requires=get_install_requires(),\n    tests_require=get_tests_require(),\n    extras_require=get_extras_require(),\n    entry_points={'console_scripts': ['optuna = optuna.cli:main']})\n", "path": "setup.py"}, {"content": "\"\"\"\nOptuna example that optimizes multi-layer perceptrons using Tensorflow (Eager Execution).\n\nIn this example, we optimize the validation accuracy of hand-written digit recognition using\nTensorflow and MNIST. We optimize the neural network architecture as well as the optimizer\nconfiguration.\n\nWe have the following two ways to execute this example:\n\n(1) Execute this code directly.\n    $ python tensorflow_eager_simple.py\n\n\n(2) Execute through CLI.\n    $ STUDY_NAME=`optuna create-study --direction maximize --storage sqlite:///example.db`\n    $ optuna study optimize tensorflow_eager_simple.py objective --n-trials=100 \\\n      --study $STUDY_NAME --storage sqlite:///example.db\n\n\"\"\"\n\nimport pkg_resources\nimport tensorflow as tf\nfrom tensorflow.keras.datasets import mnist\n\nimport optuna\n\nif pkg_resources.parse_version(tf.__version__) < pkg_resources.parse_version('2.0.0'):\n    raise RuntimeError('tensorflow>=2.0.0 is required for this example.')\n\nN_TRAIN_EXAMPLES = 3000\nN_TEST_EXAMPLES = 1000\nBATCHSIZE = 128\nCLASSES = 10\nEPOCHS = 1\n\n\ndef create_model(trial):\n    # We optimize the numbers of layers, their units and weight decay parameter.\n    n_layers = trial.suggest_int('n_layers', 1, 3)\n    weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)\n    model = tf.keras.Sequential()\n    model.add(tf.keras.layers.Flatten())\n    for i in range(n_layers):\n        num_hidden = int(trial.suggest_loguniform('n_units_l{}'.format(i), 4, 128))\n        model.add(\n            tf.keras.layers.Dense(num_hidden,\n                                  activation='relu',\n                                  kernel_regularizer=tf.keras.regularizers.l2(weight_decay)))\n    model.add(\n        tf.keras.layers.Dense(CLASSES, kernel_regularizer=tf.keras.regularizers.l2(weight_decay)))\n    return model\n\n\ndef create_optimizer(trial):\n    # We optimize the choice of optimizers as well as their parameters.\n    kwargs = {}\n    optimizer_options = ['RMSprop', 'Adam', 'SGD']\n    optimizer_selected = trial.suggest_categorical('optimizer', optimizer_options)\n    if optimizer_selected == 'RMSprop':\n        kwargs['learning_rate'] = trial.suggest_loguniform('rmsprop_learning_rate', 1e-5, 1e-1)\n        kwargs['decay'] = trial.suggest_uniform('rmsprop_decay', 0.85, 0.99)\n        kwargs['momentum'] = trial.suggest_loguniform('rmsprop_momentum', 1e-5, 1e-1)\n    elif optimizer_selected == 'Adam':\n        kwargs['learning_rate'] = trial.suggest_loguniform('adam_learning_rate', 1e-5, 1e-1)\n    elif optimizer_selected == 'SGD':\n        kwargs['learning_rate'] = trial.suggest_loguniform('sgd_opt_learning_rate', 1e-5,\n                                                           1e-1)\n        kwargs['momentum'] = trial.suggest_loguniform('sgd_opt_momentum', 1e-5, 1e-1)\n\n    optimizer = getattr(tf.optimizers, optimizer_selected)(**kwargs)\n    return optimizer\n\n\ndef learn(model, optimizer, dataset, mode='eval'):\n    accuracy = tf.metrics.Accuracy('accuracy', dtype=tf.float32)\n\n    for batch, (images, labels) in enumerate(dataset):\n        with tf.GradientTape() as tape:\n            logits = model(images, training=(mode == 'train'))\n            loss_value = tf.reduce_mean(\n                tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels))\n            if mode == 'eval':\n                accuracy(tf.argmax(logits, axis=1, output_type=tf.int64),\n                         tf.cast(labels, tf.int64))\n            else:\n                grads = tape.gradient(loss_value, model.variables)\n                optimizer.apply_gradients(zip(grads, model.variables))\n\n    if mode == 'eval':\n        return accuracy\n\n\ndef get_mnist():\n    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n    x_train = x_train.astype('float32') / 255\n    x_test = x_test.astype('float32') / 255\n\n    y_train = y_train.astype('int32')\n    y_test = y_test.astype('int32')\n\n    train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n    train_ds = train_ds.shuffle(60000).batch(BATCHSIZE).take(N_TRAIN_EXAMPLES)\n\n    test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n    test_ds = test_ds.shuffle(10000).batch(BATCHSIZE).take(N_TEST_EXAMPLES)\n    return train_ds, test_ds\n\n\n# FYI: Objective functions can take additional arguments\n# (https://optuna.readthedocs.io/en/stable/faq.html#objective-func-additional-args).\ndef objective(trial):\n    # Get MNIST data.\n    train_ds, test_ds = get_mnist()\n\n    # Build model and optimizer.\n    model = create_model(trial)\n    optimizer = create_optimizer(trial)\n\n    # Training and validating cycle.\n    with tf.device(\"/cpu:0\"):\n        for _ in range(EPOCHS):\n            learn(model, optimizer, train_ds, 'train')\n\n        accuracy = learn(model, optimizer, test_ds, 'eval')\n\n    # Return last validation accuracy.\n    return accuracy.result()\n\n\nif __name__ == '__main__':\n    study = optuna.create_study(direction='maximize')\n    study.optimize(objective, n_trials=100)\n\n    print('Number of finished trials: ', len(study.trials))\n\n    print('Best trial:')\n    trial = study.best_trial\n\n    print('  Value: ', trial.value)\n\n    print('  Params: ')\n    for key, value in trial.params.items():\n        print('    {}: {}'.format(key, value))\n", "path": "examples/tensorflow_eager_simple.py"}]}