We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
ChainerMN's ImageNet example is broken because of update in pickle module
* Conditions
<!-- If you're using Chainer 4.0+, you can also get this information by typing `python -c 'import chainer; chainer.print_runtime_info()'. -->
  - Chainer >= 5.0.0

* Code to reproduce
The ImageNet example on Python >= 3.6

```
import multiprocessing
p = multiprocessing.Process(target=lambda *x: x, args=())
p.start()
```

* Error messages, stack traces, or logs

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/process.py", line 105, in start
    self._popen = self._Popen(self)
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/context.py", line 223, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/context.py", line 291, in _Popen
    return Popen(process_obj)
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/popen_forkserver.py", line 35, in __init__
    super().__init__(process_obj)
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/popen_fork.py", line 20, in __init__
    self._launch(process_obj)
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/popen_forkserver.py", line 47, in _launch
    reduction.dump(process_obj, buf)
  File "/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
_pickle.PicklingError: Can't pickle <function <lambda> at 0x7fb633f35e18>: attribute lookup <lambda> on __main__ failed
```
For details, see https://docs.python.org/3.6/library/pickle.html

```
The following types can be pickled:

* None, True, and False
* integers, floating point numbers, complex numbers
* strings, bytes, bytearrays
* tuples, lists, sets, and dictionaries containing only picklable objects
* functions defined at the top level of a module (using def, not lambda)
* built-in functions defined at the top level of a module
* classes that are defined at the top level of a module
* instances of such classes whose __dict__ or the result of calling __getstate__() is picklable (see section Pickling Class Instances for details).
```

Thus lambda is not pickable.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/chainermn/imagenet/train_imagenet.py`
Content:
```
1 #!/usr/bin/env python
2 
3 from __future__ import print_function
4 import argparse
5 import multiprocessing
6 import random
7 import sys
8 
9 import numpy as np
10 
11 import chainer
12 import chainer.cuda
13 from chainer import training
14 from chainer.training import extensions
15 
16 import chainermn
17 
18 
19 import models.alex as alex
20 import models.googlenet as googlenet
21 import models.googlenetbn as googlenetbn
22 import models.nin as nin
23 import models.resnet50 as resnet50
24 
25 # Check Python version if it supports multiprocessing.set_start_method,
26 # which was introduced in Python 3.4
27 major, minor, _, _, _ = sys.version_info
28 if major <= 2 or (major == 3 and minor < 4):
29     sys.stderr.write('Error: ImageNet example uses '
30                      'chainer.iterators.MultiprocessIterator, '
31                      'which works only with Python >= 3.4. \n'
32                      'For more details, see '
33                      'http://chainermn.readthedocs.io/en/master/'
34                      'tutorial/tips_faqs.html#using-multiprocessiterator\n')
35     exit(-1)
36 
37 
38 class PreprocessedDataset(chainer.dataset.DatasetMixin):
39 
40     def __init__(self, path, root, mean, crop_size, random=True):
41         self.base = chainer.datasets.LabeledImageDataset(path, root)
42         self.mean = mean.astype(np.float32)
43         self.crop_size = crop_size
44         self.random = random
45 
46     def __len__(self):
47         return len(self.base)
48 
49     def get_example(self, i):
50         # It reads the i-th image/label pair and return a preprocessed image.
51         # It applies following preprocesses:
52         #     - Cropping (random or center rectangular)
53         #     - Random flip
54         #     - Scaling to [0, 1] value
55         crop_size = self.crop_size
56 
57         image, label = self.base[i]
58         _, h, w = image.shape
59 
60         if self.random:
61             # Randomly crop a region and flip the image
62             top = random.randint(0, h - crop_size - 1)
63             left = random.randint(0, w - crop_size - 1)
64             if random.randint(0, 1):
65                 image = image[:, :, ::-1]
66         else:
67             # Crop the center
68             top = (h - crop_size) // 2
69             left = (w - crop_size) // 2
70         bottom = top + crop_size
71         right = left + crop_size
72 
73         image = image[:, top:bottom, left:right]
74         image -= self.mean[:, top:bottom, left:right]
75         image *= (1.0 / 255.0)  # Scale to [0, 1]
76         return image, label
77 
78 
79 # chainermn.create_multi_node_evaluator can be also used with user customized
80 # evaluator classes that inherit chainer.training.extensions.Evaluator.
81 class TestModeEvaluator(extensions.Evaluator):
82 
83     def evaluate(self):
84         model = self.get_target('main')
85         model.train = False
86         ret = super(TestModeEvaluator, self).evaluate()
87         model.train = True
88         return ret
89 
90 
91 def main():
92     # Check if GPU is available
93     # (ImageNet example does not support CPU execution)
94     if not chainer.cuda.available:
95         raise RuntimeError('ImageNet requires GPU support.')
96 
97     archs = {
98         'alex': alex.Alex,
99         'googlenet': googlenet.GoogLeNet,
100         'googlenetbn': googlenetbn.GoogLeNetBN,
101         'nin': nin.NIN,
102         'resnet50': resnet50.ResNet50,
103     }
104 
105     parser = argparse.ArgumentParser(
106         description='Learning convnet from ILSVRC2012 dataset')
107     parser.add_argument('train', help='Path to training image-label list file')
108     parser.add_argument('val', help='Path to validation image-label list file')
109     parser.add_argument('--arch', '-a', choices=archs.keys(), default='nin',
110                         help='Convnet architecture')
111     parser.add_argument('--batchsize', '-B', type=int, default=32,
112                         help='Learning minibatch size')
113     parser.add_argument('--epoch', '-E', type=int, default=10,
114                         help='Number of epochs to train')
115     parser.add_argument('--initmodel',
116                         help='Initialize the model from given file')
117     parser.add_argument('--loaderjob', '-j', type=int,
118                         help='Number of parallel data loading processes')
119     parser.add_argument('--mean', '-m', default='mean.npy',
120                         help='Mean file (computed by compute_mean.py)')
121     parser.add_argument('--resume', '-r', default='',
122                         help='Initialize the trainer from given file')
123     parser.add_argument('--out', '-o', default='result',
124                         help='Output directory')
125     parser.add_argument('--root', '-R', default='.',
126                         help='Root directory path of image files')
127     parser.add_argument('--val_batchsize', '-b', type=int, default=250,
128                         help='Validation minibatch size')
129     parser.add_argument('--test', action='store_true')
130     parser.add_argument('--communicator', default='hierarchical')
131     parser.set_defaults(test=False)
132     args = parser.parse_args()
133 
134     # Start method of multiprocessing module need to be changed if we
135     # are using InfiniBand and MultiprocessIterator. This is because
136     # processes often crash when calling fork if they are using
137     # Infiniband.  (c.f.,
138     # https://www.open-mpi.org/faq/?category=tuning#fork-warning )
139     # Also, just setting the start method does not seem to be
140     # sufficient to actually launch the forkserver processes, so also
141     # start a dummy process.
142     # See also our document:
143     # https://chainermn.readthedocs.io/en/stable/tutorial/tips_faqs.html#using-multiprocessiterator
144     # This must be done *before* ``chainermn.create_communicator``!!!
145     multiprocessing.set_start_method('forkserver')
146     p = multiprocessing.Process(target=lambda *x: x, args=())
147     p.start()
148     p.join()
149 
150     # Prepare ChainerMN communicator.
151     comm = chainermn.create_communicator(args.communicator)
152     device = comm.intra_rank
153 
154     if comm.rank == 0:
155         print('==========================================')
156         print('Num process (COMM_WORLD): {}'.format(comm.size))
157         print('Using {} communicator'.format(args.communicator))
158         print('Using {} arch'.format(args.arch))
159         print('Num Minibatch-size: {}'.format(args.batchsize))
160         print('Num epoch: {}'.format(args.epoch))
161         print('==========================================')
162 
163     model = archs[args.arch]()
164     if args.initmodel:
165         print('Load model from', args.initmodel)
166         chainer.serializers.load_npz(args.initmodel, model)
167 
168     chainer.cuda.get_device_from_id(device).use()  # Make the GPU current
169     model.to_gpu()
170 
171     # Split and distribute the dataset. Only worker 0 loads the whole dataset.
172     # Datasets of worker 0 are evenly split and distributed to all workers.
173     mean = np.load(args.mean)
174     if comm.rank == 0:
175         train = PreprocessedDataset(args.train, args.root, mean, model.insize)
176         val = PreprocessedDataset(
177             args.val, args.root, mean, model.insize, False)
178     else:
179         train = None
180         val = None
181     train = chainermn.scatter_dataset(train, comm, shuffle=True)
182     val = chainermn.scatter_dataset(val, comm)
183 
184     # A workaround for processes crash should be done before making
185     # communicator above, when using fork (e.g. MultiProcessIterator)
186     # along with Infiniband.
187     train_iter = chainer.iterators.MultiprocessIterator(
188         train, args.batchsize, n_processes=args.loaderjob)
189     val_iter = chainer.iterators.MultiprocessIterator(
190         val, args.val_batchsize, repeat=False, n_processes=args.loaderjob)
191 
192     # Create a multi node optimizer from a standard Chainer optimizer.
193     optimizer = chainermn.create_multi_node_optimizer(
194         chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9), comm)
195     optimizer.setup(model)
196 
197     # Set up a trainer
198     updater = training.StandardUpdater(train_iter, optimizer, device=device)
199     trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out)
200 
201     checkpoint_interval = (10, 'iteration') if args.test else (1, 'epoch')
202     val_interval = (10, 'iteration') if args.test else (1, 'epoch')
203     log_interval = (10, 'iteration') if args.test else (1, 'epoch')
204 
205     checkpointer = chainermn.create_multi_node_checkpointer(
206         name='imagenet-example', comm=comm)
207     checkpointer.maybe_load(trainer, optimizer)
208     trainer.extend(checkpointer, trigger=checkpoint_interval)
209 
210     # Create a multi node evaluator from an evaluator.
211     evaluator = TestModeEvaluator(val_iter, model, device=device)
212     evaluator = chainermn.create_multi_node_evaluator(evaluator, comm)
213     trainer.extend(evaluator, trigger=val_interval)
214 
215     # Some display and output extensions are necessary only for one worker.
216     # (Otherwise, there would just be repeated outputs.)
217     if comm.rank == 0:
218         trainer.extend(extensions.DumpGraph('main/loss'))
219         trainer.extend(extensions.LogReport(trigger=log_interval))
220         trainer.extend(extensions.observe_lr(), trigger=log_interval)
221         trainer.extend(extensions.PrintReport([
222             'epoch', 'iteration', 'main/loss', 'validation/main/loss',
223             'main/accuracy', 'validation/main/accuracy', 'lr'
224         ]), trigger=log_interval)
225         trainer.extend(extensions.ProgressBar(update_interval=10))
226 
227     if args.resume:
228         chainer.serializers.load_npz(args.resume, trainer)
229 
230     trainer.run()
231 
232 
233 if __name__ == '__main__':
234     main()
235
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/chainermn/imagenet/train_imagenet.py b/examples/chainermn/imagenet/train_imagenet.py
--- a/examples/chainermn/imagenet/train_imagenet.py
+++ b/examples/chainermn/imagenet/train_imagenet.py
@@ -143,7 +143,7 @@
     # https://chainermn.readthedocs.io/en/stable/tutorial/tips_faqs.html#using-multiprocessiterator
     # This must be done *before* ``chainermn.create_communicator``!!!
     multiprocessing.set_start_method('forkserver')
-    p = multiprocessing.Process(target=lambda *x: x, args=())
+    p = multiprocessing.Process()
     p.start()
     p.join()

{"golden_diff": "diff --git a/examples/chainermn/imagenet/train_imagenet.py b/examples/chainermn/imagenet/train_imagenet.py\n--- a/examples/chainermn/imagenet/train_imagenet.py\n+++ b/examples/chainermn/imagenet/train_imagenet.py\n@@ -143,7 +143,7 @@\n     # https://chainermn.readthedocs.io/en/stable/tutorial/tips_faqs.html#using-multiprocessiterator\n     # This must be done *before* ``chainermn.create_communicator``!!!\n     multiprocessing.set_start_method('forkserver')\n-    p = multiprocessing.Process(target=lambda *x: x, args=())\n+    p = multiprocessing.Process()\n     p.start()\n     p.join()\n", "issue": "ChainerMN's ImageNet example is broken because of update in pickle module\n* Conditions\r\n<!-- If you're using Chainer 4.0+, you can also get this information by typing `python -c 'import chainer; chainer.print_runtime_info()'. -->\r\n  - Chainer >= 5.0.0\r\n\r\n* Code to reproduce\r\nThe ImageNet example on Python >= 3.6\r\n\r\n```\r\nimport multiprocessing\r\np = multiprocessing.Process(target=lambda *x: x, args=())\r\np.start()\r\n```\r\n\r\n* Error messages, stack traces, or logs\r\n\r\n```\r\nTraceback (most recent call last):\r\n  File \"<stdin>\", line 1, in <module>\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/process.py\", line 105, in start\r\n    self._popen = self._Popen(self)\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/context.py\", line 223, in _Popen\r\n    return _default_context.get_context().Process._Popen(process_obj)\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/context.py\", line 291, in _Popen\r\n    return Popen(process_obj)\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/popen_forkserver.py\", line 35, in __init__\r\n    super().__init__(process_obj)\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/popen_fork.py\", line 20, in __init__\r\n    self._launch(process_obj)\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/popen_forkserver.py\", line 47, in _launch\r\n    reduction.dump(process_obj, buf)\r\n  File \"/home/kfukuda/.pyenv/versions/anaconda3-4.3.1/lib/python3.6/multiprocessing/reduction.py\", line 60, in dump\r\n    ForkingPickler(file, protocol).dump(obj)\r\n_pickle.PicklingError: Can't pickle <function <lambda> at 0x7fb633f35e18>: attribute lookup <lambda> on __main__ failed\r\n```\r\nFor details, see https://docs.python.org/3.6/library/pickle.html\r\n\r\n```\r\nThe following types can be pickled:\r\n\r\n* None, True, and False\r\n* integers, floating point numbers, complex numbers\r\n* strings, bytes, bytearrays\r\n* tuples, lists, sets, and dictionaries containing only picklable objects\r\n* functions defined at the top level of a module (using def, not lambda)\r\n* built-in functions defined at the top level of a module\r\n* classes that are defined at the top level of a module\r\n* instances of such classes whose __dict__ or the result of calling __getstate__() is picklable (see section Pickling Class Instances for details).\r\n```\r\n\r\nThus lambda is not pickable.\r\n\n", "before_files": [{"content": "#!/usr/bin/env python\n\nfrom __future__ import print_function\nimport argparse\nimport multiprocessing\nimport random\nimport sys\n\nimport numpy as np\n\nimport chainer\nimport chainer.cuda\nfrom chainer import training\nfrom chainer.training import extensions\n\nimport chainermn\n\n\nimport models.alex as alex\nimport models.googlenet as googlenet\nimport models.googlenetbn as googlenetbn\nimport models.nin as nin\nimport models.resnet50 as resnet50\n\n# Check Python version if it supports multiprocessing.set_start_method,\n# which was introduced in Python 3.4\nmajor, minor, _, _, _ = sys.version_info\nif major <= 2 or (major == 3 and minor < 4):\n    sys.stderr.write('Error: ImageNet example uses '\n                     'chainer.iterators.MultiprocessIterator, '\n                     'which works only with Python >= 3.4. \\n'\n                     'For more details, see '\n                     'http://chainermn.readthedocs.io/en/master/'\n                     'tutorial/tips_faqs.html#using-multiprocessiterator\\n')\n    exit(-1)\n\n\nclass PreprocessedDataset(chainer.dataset.DatasetMixin):\n\n    def __init__(self, path, root, mean, crop_size, random=True):\n        self.base = chainer.datasets.LabeledImageDataset(path, root)\n        self.mean = mean.astype(np.float32)\n        self.crop_size = crop_size\n        self.random = random\n\n    def __len__(self):\n        return len(self.base)\n\n    def get_example(self, i):\n        # It reads the i-th image/label pair and return a preprocessed image.\n        # It applies following preprocesses:\n        #     - Cropping (random or center rectangular)\n        #     - Random flip\n        #     - Scaling to [0, 1] value\n        crop_size = self.crop_size\n\n        image, label = self.base[i]\n        _, h, w = image.shape\n\n        if self.random:\n            # Randomly crop a region and flip the image\n            top = random.randint(0, h - crop_size - 1)\n            left = random.randint(0, w - crop_size - 1)\n            if random.randint(0, 1):\n                image = image[:, :, ::-1]\n        else:\n            # Crop the center\n            top = (h - crop_size) // 2\n            left = (w - crop_size) // 2\n        bottom = top + crop_size\n        right = left + crop_size\n\n        image = image[:, top:bottom, left:right]\n        image -= self.mean[:, top:bottom, left:right]\n        image *= (1.0 / 255.0)  # Scale to [0, 1]\n        return image, label\n\n\n# chainermn.create_multi_node_evaluator can be also used with user customized\n# evaluator classes that inherit chainer.training.extensions.Evaluator.\nclass TestModeEvaluator(extensions.Evaluator):\n\n    def evaluate(self):\n        model = self.get_target('main')\n        model.train = False\n        ret = super(TestModeEvaluator, self).evaluate()\n        model.train = True\n        return ret\n\n\ndef main():\n    # Check if GPU is available\n    # (ImageNet example does not support CPU execution)\n    if not chainer.cuda.available:\n        raise RuntimeError('ImageNet requires GPU support.')\n\n    archs = {\n        'alex': alex.Alex,\n        'googlenet': googlenet.GoogLeNet,\n        'googlenetbn': googlenetbn.GoogLeNetBN,\n        'nin': nin.NIN,\n        'resnet50': resnet50.ResNet50,\n    }\n\n    parser = argparse.ArgumentParser(\n        description='Learning convnet from ILSVRC2012 dataset')\n    parser.add_argument('train', help='Path to training image-label list file')\n    parser.add_argument('val', help='Path to validation image-label list file')\n    parser.add_argument('--arch', '-a', choices=archs.keys(), default='nin',\n                        help='Convnet architecture')\n    parser.add_argument('--batchsize', '-B', type=int, default=32,\n                        help='Learning minibatch size')\n    parser.add_argument('--epoch', '-E', type=int, default=10,\n                        help='Number of epochs to train')\n    parser.add_argument('--initmodel',\n                        help='Initialize the model from given file')\n    parser.add_argument('--loaderjob', '-j', type=int,\n                        help='Number of parallel data loading processes')\n    parser.add_argument('--mean', '-m', default='mean.npy',\n                        help='Mean file (computed by compute_mean.py)')\n    parser.add_argument('--resume', '-r', default='',\n                        help='Initialize the trainer from given file')\n    parser.add_argument('--out', '-o', default='result',\n                        help='Output directory')\n    parser.add_argument('--root', '-R', default='.',\n                        help='Root directory path of image files')\n    parser.add_argument('--val_batchsize', '-b', type=int, default=250,\n                        help='Validation minibatch size')\n    parser.add_argument('--test', action='store_true')\n    parser.add_argument('--communicator', default='hierarchical')\n    parser.set_defaults(test=False)\n    args = parser.parse_args()\n\n    # Start method of multiprocessing module need to be changed if we\n    # are using InfiniBand and MultiprocessIterator. This is because\n    # processes often crash when calling fork if they are using\n    # Infiniband.  (c.f.,\n    # https://www.open-mpi.org/faq/?category=tuning#fork-warning )\n    # Also, just setting the start method does not seem to be\n    # sufficient to actually launch the forkserver processes, so also\n    # start a dummy process.\n    # See also our document:\n    # https://chainermn.readthedocs.io/en/stable/tutorial/tips_faqs.html#using-multiprocessiterator\n    # This must be done *before* ``chainermn.create_communicator``!!!\n    multiprocessing.set_start_method('forkserver')\n    p = multiprocessing.Process(target=lambda *x: x, args=())\n    p.start()\n    p.join()\n\n    # Prepare ChainerMN communicator.\n    comm = chainermn.create_communicator(args.communicator)\n    device = comm.intra_rank\n\n    if comm.rank == 0:\n        print('==========================================')\n        print('Num process (COMM_WORLD): {}'.format(comm.size))\n        print('Using {} communicator'.format(args.communicator))\n        print('Using {} arch'.format(args.arch))\n        print('Num Minibatch-size: {}'.format(args.batchsize))\n        print('Num epoch: {}'.format(args.epoch))\n        print('==========================================')\n\n    model = archs[args.arch]()\n    if args.initmodel:\n        print('Load model from', args.initmodel)\n        chainer.serializers.load_npz(args.initmodel, model)\n\n    chainer.cuda.get_device_from_id(device).use()  # Make the GPU current\n    model.to_gpu()\n\n    # Split and distribute the dataset. Only worker 0 loads the whole dataset.\n    # Datasets of worker 0 are evenly split and distributed to all workers.\n    mean = np.load(args.mean)\n    if comm.rank == 0:\n        train = PreprocessedDataset(args.train, args.root, mean, model.insize)\n        val = PreprocessedDataset(\n            args.val, args.root, mean, model.insize, False)\n    else:\n        train = None\n        val = None\n    train = chainermn.scatter_dataset(train, comm, shuffle=True)\n    val = chainermn.scatter_dataset(val, comm)\n\n    # A workaround for processes crash should be done before making\n    # communicator above, when using fork (e.g. MultiProcessIterator)\n    # along with Infiniband.\n    train_iter = chainer.iterators.MultiprocessIterator(\n        train, args.batchsize, n_processes=args.loaderjob)\n    val_iter = chainer.iterators.MultiprocessIterator(\n        val, args.val_batchsize, repeat=False, n_processes=args.loaderjob)\n\n    # Create a multi node optimizer from a standard Chainer optimizer.\n    optimizer = chainermn.create_multi_node_optimizer(\n        chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9), comm)\n    optimizer.setup(model)\n\n    # Set up a trainer\n    updater = training.StandardUpdater(train_iter, optimizer, device=device)\n    trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out)\n\n    checkpoint_interval = (10, 'iteration') if args.test else (1, 'epoch')\n    val_interval = (10, 'iteration') if args.test else (1, 'epoch')\n    log_interval = (10, 'iteration') if args.test else (1, 'epoch')\n\n    checkpointer = chainermn.create_multi_node_checkpointer(\n        name='imagenet-example', comm=comm)\n    checkpointer.maybe_load(trainer, optimizer)\n    trainer.extend(checkpointer, trigger=checkpoint_interval)\n\n    # Create a multi node evaluator from an evaluator.\n    evaluator = TestModeEvaluator(val_iter, model, device=device)\n    evaluator = chainermn.create_multi_node_evaluator(evaluator, comm)\n    trainer.extend(evaluator, trigger=val_interval)\n\n    # Some display and output extensions are necessary only for one worker.\n    # (Otherwise, there would just be repeated outputs.)\n    if comm.rank == 0:\n        trainer.extend(extensions.DumpGraph('main/loss'))\n        trainer.extend(extensions.LogReport(trigger=log_interval))\n        trainer.extend(extensions.observe_lr(), trigger=log_interval)\n        trainer.extend(extensions.PrintReport([\n            'epoch', 'iteration', 'main/loss', 'validation/main/loss',\n            'main/accuracy', 'validation/main/accuracy', 'lr'\n        ]), trigger=log_interval)\n        trainer.extend(extensions.ProgressBar(update_interval=10))\n\n    if args.resume:\n        chainer.serializers.load_npz(args.resume, trainer)\n\n    trainer.run()\n\n\nif __name__ == '__main__':\n    main()\n", "path": "examples/chainermn/imagenet/train_imagenet.py"}], "after_files": [{"content": "#!/usr/bin/env python\n\nfrom __future__ import print_function\nimport argparse\nimport multiprocessing\nimport random\nimport sys\n\nimport numpy as np\n\nimport chainer\nimport chainer.cuda\nfrom chainer import training\nfrom chainer.training import extensions\n\nimport chainermn\n\n\nimport models.alex as alex\nimport models.googlenet as googlenet\nimport models.googlenetbn as googlenetbn\nimport models.nin as nin\nimport models.resnet50 as resnet50\n\n# Check Python version if it supports multiprocessing.set_start_method,\n# which was introduced in Python 3.4\nmajor, minor, _, _, _ = sys.version_info\nif major <= 2 or (major == 3 and minor < 4):\n    sys.stderr.write('Error: ImageNet example uses '\n                     'chainer.iterators.MultiprocessIterator, '\n                     'which works only with Python >= 3.4. \\n'\n                     'For more details, see '\n                     'http://chainermn.readthedocs.io/en/master/'\n                     'tutorial/tips_faqs.html#using-multiprocessiterator\\n')\n    exit(-1)\n\n\nclass PreprocessedDataset(chainer.dataset.DatasetMixin):\n\n    def __init__(self, path, root, mean, crop_size, random=True):\n        self.base = chainer.datasets.LabeledImageDataset(path, root)\n        self.mean = mean.astype(np.float32)\n        self.crop_size = crop_size\n        self.random = random\n\n    def __len__(self):\n        return len(self.base)\n\n    def get_example(self, i):\n        # It reads the i-th image/label pair and return a preprocessed image.\n        # It applies following preprocesses:\n        #     - Cropping (random or center rectangular)\n        #     - Random flip\n        #     - Scaling to [0, 1] value\n        crop_size = self.crop_size\n\n        image, label = self.base[i]\n        _, h, w = image.shape\n\n        if self.random:\n            # Randomly crop a region and flip the image\n            top = random.randint(0, h - crop_size - 1)\n            left = random.randint(0, w - crop_size - 1)\n            if random.randint(0, 1):\n                image = image[:, :, ::-1]\n        else:\n            # Crop the center\n            top = (h - crop_size) // 2\n            left = (w - crop_size) // 2\n        bottom = top + crop_size\n        right = left + crop_size\n\n        image = image[:, top:bottom, left:right]\n        image -= self.mean[:, top:bottom, left:right]\n        image *= (1.0 / 255.0)  # Scale to [0, 1]\n        return image, label\n\n\n# chainermn.create_multi_node_evaluator can be also used with user customized\n# evaluator classes that inherit chainer.training.extensions.Evaluator.\nclass TestModeEvaluator(extensions.Evaluator):\n\n    def evaluate(self):\n        model = self.get_target('main')\n        model.train = False\n        ret = super(TestModeEvaluator, self).evaluate()\n        model.train = True\n        return ret\n\n\ndef main():\n    # Check if GPU is available\n    # (ImageNet example does not support CPU execution)\n    if not chainer.cuda.available:\n        raise RuntimeError('ImageNet requires GPU support.')\n\n    archs = {\n        'alex': alex.Alex,\n        'googlenet': googlenet.GoogLeNet,\n        'googlenetbn': googlenetbn.GoogLeNetBN,\n        'nin': nin.NIN,\n        'resnet50': resnet50.ResNet50,\n    }\n\n    parser = argparse.ArgumentParser(\n        description='Learning convnet from ILSVRC2012 dataset')\n    parser.add_argument('train', help='Path to training image-label list file')\n    parser.add_argument('val', help='Path to validation image-label list file')\n    parser.add_argument('--arch', '-a', choices=archs.keys(), default='nin',\n                        help='Convnet architecture')\n    parser.add_argument('--batchsize', '-B', type=int, default=32,\n                        help='Learning minibatch size')\n    parser.add_argument('--epoch', '-E', type=int, default=10,\n                        help='Number of epochs to train')\n    parser.add_argument('--initmodel',\n                        help='Initialize the model from given file')\n    parser.add_argument('--loaderjob', '-j', type=int,\n                        help='Number of parallel data loading processes')\n    parser.add_argument('--mean', '-m', default='mean.npy',\n                        help='Mean file (computed by compute_mean.py)')\n    parser.add_argument('--resume', '-r', default='',\n                        help='Initialize the trainer from given file')\n    parser.add_argument('--out', '-o', default='result',\n                        help='Output directory')\n    parser.add_argument('--root', '-R', default='.',\n                        help='Root directory path of image files')\n    parser.add_argument('--val_batchsize', '-b', type=int, default=250,\n                        help='Validation minibatch size')\n    parser.add_argument('--test', action='store_true')\n    parser.add_argument('--communicator', default='hierarchical')\n    parser.set_defaults(test=False)\n    args = parser.parse_args()\n\n    # Start method of multiprocessing module need to be changed if we\n    # are using InfiniBand and MultiprocessIterator. This is because\n    # processes often crash when calling fork if they are using\n    # Infiniband.  (c.f.,\n    # https://www.open-mpi.org/faq/?category=tuning#fork-warning )\n    # Also, just setting the start method does not seem to be\n    # sufficient to actually launch the forkserver processes, so also\n    # start a dummy process.\n    # See also our document:\n    # https://chainermn.readthedocs.io/en/stable/tutorial/tips_faqs.html#using-multiprocessiterator\n    # This must be done *before* ``chainermn.create_communicator``!!!\n    multiprocessing.set_start_method('forkserver')\n    p = multiprocessing.Process()\n    p.start()\n    p.join()\n\n    # Prepare ChainerMN communicator.\n    comm = chainermn.create_communicator(args.communicator)\n    device = comm.intra_rank\n\n    if comm.rank == 0:\n        print('==========================================')\n        print('Num process (COMM_WORLD): {}'.format(comm.size))\n        print('Using {} communicator'.format(args.communicator))\n        print('Using {} arch'.format(args.arch))\n        print('Num Minibatch-size: {}'.format(args.batchsize))\n        print('Num epoch: {}'.format(args.epoch))\n        print('==========================================')\n\n    model = archs[args.arch]()\n    if args.initmodel:\n        print('Load model from', args.initmodel)\n        chainer.serializers.load_npz(args.initmodel, model)\n\n    chainer.cuda.get_device_from_id(device).use()  # Make the GPU current\n    model.to_gpu()\n\n    # Split and distribute the dataset. Only worker 0 loads the whole dataset.\n    # Datasets of worker 0 are evenly split and distributed to all workers.\n    mean = np.load(args.mean)\n    if comm.rank == 0:\n        train = PreprocessedDataset(args.train, args.root, mean, model.insize)\n        val = PreprocessedDataset(\n            args.val, args.root, mean, model.insize, False)\n    else:\n        train = None\n        val = None\n    train = chainermn.scatter_dataset(train, comm, shuffle=True)\n    val = chainermn.scatter_dataset(val, comm)\n\n    # A workaround for processes crash should be done before making\n    # communicator above, when using fork (e.g. MultiProcessIterator)\n    # along with Infiniband.\n    train_iter = chainer.iterators.MultiprocessIterator(\n        train, args.batchsize, n_processes=args.loaderjob)\n    val_iter = chainer.iterators.MultiprocessIterator(\n        val, args.val_batchsize, repeat=False, n_processes=args.loaderjob)\n\n    # Create a multi node optimizer from a standard Chainer optimizer.\n    optimizer = chainermn.create_multi_node_optimizer(\n        chainer.optimizers.MomentumSGD(lr=0.01, momentum=0.9), comm)\n    optimizer.setup(model)\n\n    # Set up a trainer\n    updater = training.StandardUpdater(train_iter, optimizer, device=device)\n    trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out)\n\n    checkpoint_interval = (10, 'iteration') if args.test else (1, 'epoch')\n    val_interval = (10, 'iteration') if args.test else (1, 'epoch')\n    log_interval = (10, 'iteration') if args.test else (1, 'epoch')\n\n    checkpointer = chainermn.create_multi_node_checkpointer(\n        name='imagenet-example', comm=comm)\n    checkpointer.maybe_load(trainer, optimizer)\n    trainer.extend(checkpointer, trigger=checkpoint_interval)\n\n    # Create a multi node evaluator from an evaluator.\n    evaluator = TestModeEvaluator(val_iter, model, device=device)\n    evaluator = chainermn.create_multi_node_evaluator(evaluator, comm)\n    trainer.extend(evaluator, trigger=val_interval)\n\n    # Some display and output extensions are necessary only for one worker.\n    # (Otherwise, there would just be repeated outputs.)\n    if comm.rank == 0:\n        trainer.extend(extensions.DumpGraph('main/loss'))\n        trainer.extend(extensions.LogReport(trigger=log_interval))\n        trainer.extend(extensions.observe_lr(), trigger=log_interval)\n        trainer.extend(extensions.PrintReport([\n            'epoch', 'iteration', 'main/loss', 'validation/main/loss',\n            'main/accuracy', 'validation/main/accuracy', 'lr'\n        ]), trigger=log_interval)\n        trainer.extend(extensions.ProgressBar(update_interval=10))\n\n    if args.resume:\n        chainer.serializers.load_npz(args.resume, trainer)\n\n    trainer.run()\n\n\nif __name__ == '__main__':\n    main()\n", "path": "examples/chainermn/imagenet/train_imagenet.py"}]}