We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
work around hung
The fix will follow closely. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `PaddleNLP/benchmark/transformer/reader.py`
Content:
```
1 # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import glob
16 import sys
17 import os
18 import io
19 import itertools
20 from functools import partial
21 
22 import numpy as np
23 from paddle.io import BatchSampler, DataLoader, Dataset
24 from paddlenlp.data import Pad
25 from paddlenlp.datasets import WMT14ende
26 from paddlenlp.data.sampler import SamplerHelper
27 
28 
29 def min_max_filer(data, max_len, min_len=0):
30     # 1 for special tokens.
31     data_min_len = min(len(data[0]), len(data[1])) + 1
32     data_max_len = max(len(data[0]), len(data[1])) + 1
33     return (data_min_len >= min_len) and (data_max_len <= max_len)
34 
35 
36 def create_data_loader(args):
37     root = None if args.root == "None" else args.root
38     (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)
39     args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)
40     transform_func = WMT14ende.get_default_transform_func(root=root)
41     datasets = [
42         WMT14ende.get_datasets(
43             mode=m, transform_func=transform_func) for m in ["train", "dev"]
44     ]
45 
46     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,
47                       data_source):
48         return max(tokens_sofar,
49                    len(data_source[current_idx][0]) + 1,
50                    len(data_source[current_idx][1]) + 1)
51 
52     def _key(size_so_far, minibatch_len):
53         return size_so_far * minibatch_len
54 
55     data_loaders = [(None)] * 2
56     for i, dataset in enumerate(datasets):
57         m = dataset.mode
58         dataset = dataset.filter(
59             partial(
60                 min_max_filer, max_len=args.max_length))
61         sampler = SamplerHelper(dataset)
62 
63         src_key = (lambda x, data_source: len(data_source[x][0]) + 1)
64         if args.sort_type == SortType.GLOBAL:
65             buffer_size = -1
66             trg_key = (lambda x, data_source: len(data_source[x][1]) + 1)
67             # Sort twice
68             sampler = sampler.sort(
69                 key=trg_key, buffer_size=buffer_size).sort(
70                     key=src_key, buffer_size=buffer_size)
71         else:
72             sampler = sampler.shuffle()
73             if args.sort_type == SortType.POOL:
74                 buffer_size = args.pool_size
75                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)
76 
77         batch_sampler = sampler.batch(
78             batch_size=args.batch_size,
79             drop_last=False,
80             batch_size_fn=_max_token_fn,
81             key=_key)
82 
83         if m == "train":
84             batch_sampler = batch_sampler.shard()
85 
86         data_loader = DataLoader(
87             dataset=dataset,
88             batch_sampler=batch_sampler,
89             collate_fn=partial(
90                 prepare_train_input,
91                 bos_idx=args.bos_idx,
92                 eos_idx=args.eos_idx,
93                 pad_idx=args.bos_idx),
94             num_workers=0,
95             return_list=True)
96         data_loaders[i] = (data_loader)
97     return data_loaders
98 
99 
100 def create_infer_loader(args):
101     root = None if args.root == "None" else args.root
102     (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)
103     args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)
104     transform_func = WMT14ende.get_default_transform_func(root=root)
105     dataset = WMT14ende.get_datasets(
106         mode="test", transform_func=transform_func).filter(
107             partial(
108                 min_max_filer, max_len=args.max_length))
109 
110     batch_sampler = SamplerHelper(dataset).batch(
111         batch_size=args.infer_batch_size, drop_last=False)
112 
113     data_loader = DataLoader(
114         dataset=dataset,
115         batch_sampler=batch_sampler,
116         collate_fn=partial(
117             prepare_infer_input,
118             bos_idx=args.bos_idx,
119             eos_idx=args.eos_idx,
120             pad_idx=args.bos_idx),
121         num_workers=0,
122         return_list=True)
123     return data_loader, trg_vocab.to_tokens
124 
125 
126 def prepare_train_input(insts, bos_idx, eos_idx, pad_idx):
127     """
128     Put all padded data needed by training into a list.
129     """
130     word_pad = Pad(pad_idx)
131     src_word = word_pad([inst[0] + [eos_idx] for inst in insts])
132     trg_word = word_pad([[bos_idx] + inst[1] for inst in insts])
133     lbl_word = np.expand_dims(
134         word_pad([inst[1] + [eos_idx] for inst in insts]), axis=2)
135 
136     data_inputs = [src_word, trg_word, lbl_word]
137 
138     return data_inputs
139 
140 
141 def prepare_infer_input(insts, bos_idx, eos_idx, pad_idx):
142     """
143     Put all padded data needed by beam search decoder into a list.
144     """
145     word_pad = Pad(pad_idx)
146     src_word = word_pad([inst[0] + [eos_idx] for inst in insts])
147 
148     return [src_word, ]
149 
150 
151 class SortType(object):
152     GLOBAL = 'global'
153     POOL = 'pool'
154     NONE = "none"
155
```

Path: `PaddleNLP/examples/machine_translation/transformer/reader.py`
Content:
```
1 # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import glob
16 import sys
17 import os
18 import io
19 import itertools
20 from functools import partial
21 
22 import numpy as np
23 from paddle.io import BatchSampler, DataLoader, Dataset
24 from paddlenlp.data import Pad
25 from paddlenlp.datasets import WMT14ende
26 from paddlenlp.data.sampler import SamplerHelper
27 
28 
29 def min_max_filer(data, max_len, min_len=0):
30     # 1 for special tokens.
31     data_min_len = min(len(data[0]), len(data[1])) + 1
32     data_max_len = max(len(data[0]), len(data[1])) + 1
33     return (data_min_len >= min_len) and (data_max_len <= max_len)
34 
35 
36 def create_data_loader(args):
37     root = None if args.root == "None" else args.root
38     (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)
39     args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)
40     transform_func = WMT14ende.get_default_transform_func(root=root)
41     datasets = [
42         WMT14ende.get_datasets(
43             mode=m, transform_func=transform_func) for m in ["train", "dev"]
44     ]
45 
46     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,
47                       data_source):
48         return max(tokens_sofar,
49                    len(data_source[current_idx][0]) + 1,
50                    len(data_source[current_idx][1]) + 1)
51 
52     def _key(size_so_far, minibatch_len):
53         return size_so_far * minibatch_len
54 
55     data_loaders = [(None)] * 2
56     for i, dataset in enumerate(datasets):
57         m = dataset.mode
58         dataset = dataset.filter(
59             partial(
60                 min_max_filer, max_len=args.max_length))
61         sampler = SamplerHelper(dataset)
62 
63         src_key = (lambda x, data_source: len(data_source[x][0]) + 1)
64         if args.sort_type == SortType.GLOBAL:
65             buffer_size = -1
66             trg_key = (lambda x, data_source: len(data_source[x][1]) + 1)
67             # Sort twice
68             sampler = sampler.sort(
69                 key=trg_key, buffer_size=buffer_size).sort(
70                     key=src_key, buffer_size=buffer_size)
71         else:
72             sampler = sampler.shuffle()
73             if args.sort_type == SortType.POOL:
74                 buffer_size = args.pool_size
75                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)
76 
77         batch_sampler = sampler.batch(
78             batch_size=args.batch_size,
79             drop_last=False,
80             batch_size_fn=_max_token_fn,
81             key=_key)
82 
83         if m == "train":
84             batch_sampler = batch_sampler.shard()
85 
86         data_loader = DataLoader(
87             dataset=dataset,
88             batch_sampler=batch_sampler,
89             collate_fn=partial(
90                 prepare_train_input,
91                 bos_idx=args.bos_idx,
92                 eos_idx=args.eos_idx,
93                 pad_idx=args.bos_idx),
94             num_workers=0,
95             return_list=True)
96         data_loaders[i] = (data_loader)
97     return data_loaders
98 
99 
100 def create_infer_loader(args):
101     root = None if args.root == "None" else args.root
102     (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)
103     args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)
104     transform_func = WMT14ende.get_default_transform_func(root=root)
105     dataset = WMT14ende.get_datasets(
106         mode="test", transform_func=transform_func).filter(
107             partial(
108                 min_max_filer, max_len=args.max_length))
109 
110     batch_sampler = SamplerHelper(dataset).batch(
111         batch_size=args.infer_batch_size, drop_last=False)
112 
113     data_loader = DataLoader(
114         dataset=dataset,
115         batch_sampler=batch_sampler,
116         collate_fn=partial(
117             prepare_infer_input,
118             bos_idx=args.bos_idx,
119             eos_idx=args.eos_idx,
120             pad_idx=args.bos_idx),
121         num_workers=0,
122         return_list=True)
123     return data_loader, trg_vocab.to_tokens
124 
125 
126 def prepare_train_input(insts, bos_idx, eos_idx, pad_idx):
127     """
128     Put all padded data needed by training into a list.
129     """
130     word_pad = Pad(pad_idx)
131     src_word = word_pad([inst[0] + [eos_idx] for inst in insts])
132     trg_word = word_pad([[bos_idx] + inst[1] for inst in insts])
133     lbl_word = np.expand_dims(
134         word_pad([inst[1] + [eos_idx] for inst in insts]), axis=2)
135 
136     data_inputs = [src_word, trg_word, lbl_word]
137 
138     return data_inputs
139 
140 
141 def prepare_infer_input(insts, bos_idx, eos_idx, pad_idx):
142     """
143     Put all padded data needed by beam search decoder into a list.
144     """
145     word_pad = Pad(pad_idx)
146     src_word = word_pad([inst[0] + [eos_idx] for inst in insts])
147 
148     return [src_word, ]
149 
150 
151 class SortType(object):
152     GLOBAL = 'global'
153     POOL = 'pool'
154     NONE = "none"
155
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/PaddleNLP/benchmark/transformer/reader.py b/PaddleNLP/benchmark/transformer/reader.py
--- a/PaddleNLP/benchmark/transformer/reader.py
+++ b/PaddleNLP/benchmark/transformer/reader.py
@@ -43,6 +43,12 @@
             mode=m, transform_func=transform_func) for m in ["train", "dev"]
     ]
 
+    if args.shuffle or args.shuffle_batch:
+        if args.shuffle_seed == "None" or args.shuffle_seed is None:
+            shuffle_seed = 0
+        else:
+            shuffle_seed = args.shuffle_seed
+
     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,
                       data_source):
         return max(tokens_sofar,
@@ -69,7 +75,8 @@
                 key=trg_key, buffer_size=buffer_size).sort(
                     key=src_key, buffer_size=buffer_size)
         else:
-            sampler = sampler.shuffle()
+            if args.shuffle:
+                sampler = sampler.shuffle(seed=shuffle_seed)
             if args.sort_type == SortType.POOL:
                 buffer_size = args.pool_size
                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)
@@ -83,6 +90,9 @@
         if m == "train":
             batch_sampler = batch_sampler.shard()
 
+        if args.shuffle_batch:
+            batch_sampler.shuffle(seed=shuffle_seed)
+
         data_loader = DataLoader(
             dataset=dataset,
             batch_sampler=batch_sampler,
diff --git a/PaddleNLP/examples/machine_translation/transformer/reader.py b/PaddleNLP/examples/machine_translation/transformer/reader.py
--- a/PaddleNLP/examples/machine_translation/transformer/reader.py
+++ b/PaddleNLP/examples/machine_translation/transformer/reader.py
@@ -43,6 +43,12 @@
             mode=m, transform_func=transform_func) for m in ["train", "dev"]
     ]
 
+    if args.shuffle or args.shuffle_batch:
+        if args.shuffle_seed == "None" or args.shuffle_seed is None:
+            shuffle_seed = 0
+        else:
+            shuffle_seed = args.shuffle_seed
+
     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,
                       data_source):
         return max(tokens_sofar,
@@ -69,7 +75,8 @@
                 key=trg_key, buffer_size=buffer_size).sort(
                     key=src_key, buffer_size=buffer_size)
         else:
-            sampler = sampler.shuffle()
+            if args.shuffle:
+                sampler = sampler.shuffle(seed=shuffle_seed)
             if args.sort_type == SortType.POOL:
                 buffer_size = args.pool_size
                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)
@@ -83,6 +90,9 @@
         if m == "train":
             batch_sampler = batch_sampler.shard()
 
+        if args.shuffle_batch:
+            batch_sampler.shuffle(seed=shuffle_seed)
+
         data_loader = DataLoader(
             dataset=dataset,
             batch_sampler=batch_sampler,

{"golden_diff": "diff --git a/PaddleNLP/benchmark/transformer/reader.py b/PaddleNLP/benchmark/transformer/reader.py\n--- a/PaddleNLP/benchmark/transformer/reader.py\n+++ b/PaddleNLP/benchmark/transformer/reader.py\n@@ -43,6 +43,12 @@\n             mode=m, transform_func=transform_func) for m in [\"train\", \"dev\"]\n     ]\n \n+    if args.shuffle or args.shuffle_batch:\n+        if args.shuffle_seed == \"None\" or args.shuffle_seed is None:\n+            shuffle_seed = 0\n+        else:\n+            shuffle_seed = args.shuffle_seed\n+\n     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,\n                       data_source):\n         return max(tokens_sofar,\n@@ -69,7 +75,8 @@\n                 key=trg_key, buffer_size=buffer_size).sort(\n                     key=src_key, buffer_size=buffer_size)\n         else:\n-            sampler = sampler.shuffle()\n+            if args.shuffle:\n+                sampler = sampler.shuffle(seed=shuffle_seed)\n             if args.sort_type == SortType.POOL:\n                 buffer_size = args.pool_size\n                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)\n@@ -83,6 +90,9 @@\n         if m == \"train\":\n             batch_sampler = batch_sampler.shard()\n \n+        if args.shuffle_batch:\n+            batch_sampler.shuffle(seed=shuffle_seed)\n+\n         data_loader = DataLoader(\n             dataset=dataset,\n             batch_sampler=batch_sampler,\ndiff --git a/PaddleNLP/examples/machine_translation/transformer/reader.py b/PaddleNLP/examples/machine_translation/transformer/reader.py\n--- a/PaddleNLP/examples/machine_translation/transformer/reader.py\n+++ b/PaddleNLP/examples/machine_translation/transformer/reader.py\n@@ -43,6 +43,12 @@\n             mode=m, transform_func=transform_func) for m in [\"train\", \"dev\"]\n     ]\n \n+    if args.shuffle or args.shuffle_batch:\n+        if args.shuffle_seed == \"None\" or args.shuffle_seed is None:\n+            shuffle_seed = 0\n+        else:\n+            shuffle_seed = args.shuffle_seed\n+\n     def _max_token_fn(current_idx, current_batch_size, tokens_sofar,\n                       data_source):\n         return max(tokens_sofar,\n@@ -69,7 +75,8 @@\n                 key=trg_key, buffer_size=buffer_size).sort(\n                     key=src_key, buffer_size=buffer_size)\n         else:\n-            sampler = sampler.shuffle()\n+            if args.shuffle:\n+                sampler = sampler.shuffle(seed=shuffle_seed)\n             if args.sort_type == SortType.POOL:\n                 buffer_size = args.pool_size\n                 sampler = sampler.sort(key=src_key, buffer_size=buffer_size)\n@@ -83,6 +90,9 @@\n         if m == \"train\":\n             batch_sampler = batch_sampler.shard()\n \n+        if args.shuffle_batch:\n+            batch_sampler.shuffle(seed=shuffle_seed)\n+\n         data_loader = DataLoader(\n             dataset=dataset,\n             batch_sampler=batch_sampler,\n", "issue": "work around hung\nThe fix will follow closely. \n", "before_files": [{"content": "# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport sys\nimport os\nimport io\nimport itertools\nfrom functools import partial\n\nimport numpy as np\nfrom paddle.io import BatchSampler, DataLoader, Dataset\nfrom paddlenlp.data import Pad\nfrom paddlenlp.datasets import WMT14ende\nfrom paddlenlp.data.sampler import SamplerHelper\n\n\ndef min_max_filer(data, max_len, min_len=0):\n    # 1 for special tokens.\n    data_min_len = min(len(data[0]), len(data[1])) + 1\n    data_max_len = max(len(data[0]), len(data[1])) + 1\n    return (data_min_len >= min_len) and (data_max_len <= max_len)\n\n\ndef create_data_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    datasets = [\n        WMT14ende.get_datasets(\n            mode=m, transform_func=transform_func) for m in [\"train\", \"dev\"]\n    ]\n\n    def _max_token_fn(current_idx, current_batch_size, tokens_sofar,\n                      data_source):\n        return max(tokens_sofar,\n                   len(data_source[current_idx][0]) + 1,\n                   len(data_source[current_idx][1]) + 1)\n\n    def _key(size_so_far, minibatch_len):\n        return size_so_far * minibatch_len\n\n    data_loaders = [(None)] * 2\n    for i, dataset in enumerate(datasets):\n        m = dataset.mode\n        dataset = dataset.filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n        sampler = SamplerHelper(dataset)\n\n        src_key = (lambda x, data_source: len(data_source[x][0]) + 1)\n        if args.sort_type == SortType.GLOBAL:\n            buffer_size = -1\n            trg_key = (lambda x, data_source: len(data_source[x][1]) + 1)\n            # Sort twice\n            sampler = sampler.sort(\n                key=trg_key, buffer_size=buffer_size).sort(\n                    key=src_key, buffer_size=buffer_size)\n        else:\n            sampler = sampler.shuffle()\n            if args.sort_type == SortType.POOL:\n                buffer_size = args.pool_size\n                sampler = sampler.sort(key=src_key, buffer_size=buffer_size)\n\n        batch_sampler = sampler.batch(\n            batch_size=args.batch_size,\n            drop_last=False,\n            batch_size_fn=_max_token_fn,\n            key=_key)\n\n        if m == \"train\":\n            batch_sampler = batch_sampler.shard()\n\n        data_loader = DataLoader(\n            dataset=dataset,\n            batch_sampler=batch_sampler,\n            collate_fn=partial(\n                prepare_train_input,\n                bos_idx=args.bos_idx,\n                eos_idx=args.eos_idx,\n                pad_idx=args.bos_idx),\n            num_workers=0,\n            return_list=True)\n        data_loaders[i] = (data_loader)\n    return data_loaders\n\n\ndef create_infer_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    dataset = WMT14ende.get_datasets(\n        mode=\"test\", transform_func=transform_func).filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n\n    batch_sampler = SamplerHelper(dataset).batch(\n        batch_size=args.infer_batch_size, drop_last=False)\n\n    data_loader = DataLoader(\n        dataset=dataset,\n        batch_sampler=batch_sampler,\n        collate_fn=partial(\n            prepare_infer_input,\n            bos_idx=args.bos_idx,\n            eos_idx=args.eos_idx,\n            pad_idx=args.bos_idx),\n        num_workers=0,\n        return_list=True)\n    return data_loader, trg_vocab.to_tokens\n\n\ndef prepare_train_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by training into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n    trg_word = word_pad([[bos_idx] + inst[1] for inst in insts])\n    lbl_word = np.expand_dims(\n        word_pad([inst[1] + [eos_idx] for inst in insts]), axis=2)\n\n    data_inputs = [src_word, trg_word, lbl_word]\n\n    return data_inputs\n\n\ndef prepare_infer_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by beam search decoder into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n\n    return [src_word, ]\n\n\nclass SortType(object):\n    GLOBAL = 'global'\n    POOL = 'pool'\n    NONE = \"none\"\n", "path": "PaddleNLP/benchmark/transformer/reader.py"}, {"content": "# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport sys\nimport os\nimport io\nimport itertools\nfrom functools import partial\n\nimport numpy as np\nfrom paddle.io import BatchSampler, DataLoader, Dataset\nfrom paddlenlp.data import Pad\nfrom paddlenlp.datasets import WMT14ende\nfrom paddlenlp.data.sampler import SamplerHelper\n\n\ndef min_max_filer(data, max_len, min_len=0):\n    # 1 for special tokens.\n    data_min_len = min(len(data[0]), len(data[1])) + 1\n    data_max_len = max(len(data[0]), len(data[1])) + 1\n    return (data_min_len >= min_len) and (data_max_len <= max_len)\n\n\ndef create_data_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    datasets = [\n        WMT14ende.get_datasets(\n            mode=m, transform_func=transform_func) for m in [\"train\", \"dev\"]\n    ]\n\n    def _max_token_fn(current_idx, current_batch_size, tokens_sofar,\n                      data_source):\n        return max(tokens_sofar,\n                   len(data_source[current_idx][0]) + 1,\n                   len(data_source[current_idx][1]) + 1)\n\n    def _key(size_so_far, minibatch_len):\n        return size_so_far * minibatch_len\n\n    data_loaders = [(None)] * 2\n    for i, dataset in enumerate(datasets):\n        m = dataset.mode\n        dataset = dataset.filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n        sampler = SamplerHelper(dataset)\n\n        src_key = (lambda x, data_source: len(data_source[x][0]) + 1)\n        if args.sort_type == SortType.GLOBAL:\n            buffer_size = -1\n            trg_key = (lambda x, data_source: len(data_source[x][1]) + 1)\n            # Sort twice\n            sampler = sampler.sort(\n                key=trg_key, buffer_size=buffer_size).sort(\n                    key=src_key, buffer_size=buffer_size)\n        else:\n            sampler = sampler.shuffle()\n            if args.sort_type == SortType.POOL:\n                buffer_size = args.pool_size\n                sampler = sampler.sort(key=src_key, buffer_size=buffer_size)\n\n        batch_sampler = sampler.batch(\n            batch_size=args.batch_size,\n            drop_last=False,\n            batch_size_fn=_max_token_fn,\n            key=_key)\n\n        if m == \"train\":\n            batch_sampler = batch_sampler.shard()\n\n        data_loader = DataLoader(\n            dataset=dataset,\n            batch_sampler=batch_sampler,\n            collate_fn=partial(\n                prepare_train_input,\n                bos_idx=args.bos_idx,\n                eos_idx=args.eos_idx,\n                pad_idx=args.bos_idx),\n            num_workers=0,\n            return_list=True)\n        data_loaders[i] = (data_loader)\n    return data_loaders\n\n\ndef create_infer_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    dataset = WMT14ende.get_datasets(\n        mode=\"test\", transform_func=transform_func).filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n\n    batch_sampler = SamplerHelper(dataset).batch(\n        batch_size=args.infer_batch_size, drop_last=False)\n\n    data_loader = DataLoader(\n        dataset=dataset,\n        batch_sampler=batch_sampler,\n        collate_fn=partial(\n            prepare_infer_input,\n            bos_idx=args.bos_idx,\n            eos_idx=args.eos_idx,\n            pad_idx=args.bos_idx),\n        num_workers=0,\n        return_list=True)\n    return data_loader, trg_vocab.to_tokens\n\n\ndef prepare_train_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by training into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n    trg_word = word_pad([[bos_idx] + inst[1] for inst in insts])\n    lbl_word = np.expand_dims(\n        word_pad([inst[1] + [eos_idx] for inst in insts]), axis=2)\n\n    data_inputs = [src_word, trg_word, lbl_word]\n\n    return data_inputs\n\n\ndef prepare_infer_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by beam search decoder into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n\n    return [src_word, ]\n\n\nclass SortType(object):\n    GLOBAL = 'global'\n    POOL = 'pool'\n    NONE = \"none\"\n", "path": "PaddleNLP/examples/machine_translation/transformer/reader.py"}], "after_files": [{"content": "# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport sys\nimport os\nimport io\nimport itertools\nfrom functools import partial\n\nimport numpy as np\nfrom paddle.io import BatchSampler, DataLoader, Dataset\nfrom paddlenlp.data import Pad\nfrom paddlenlp.datasets import WMT14ende\nfrom paddlenlp.data.sampler import SamplerHelper\n\n\ndef min_max_filer(data, max_len, min_len=0):\n    # 1 for special tokens.\n    data_min_len = min(len(data[0]), len(data[1])) + 1\n    data_max_len = max(len(data[0]), len(data[1])) + 1\n    return (data_min_len >= min_len) and (data_max_len <= max_len)\n\n\ndef create_data_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    datasets = [\n        WMT14ende.get_datasets(\n            mode=m, transform_func=transform_func) for m in [\"train\", \"dev\"]\n    ]\n\n    if args.shuffle or args.shuffle_batch:\n        if args.shuffle_seed == \"None\" or args.shuffle_seed is None:\n            shuffle_seed = 0\n        else:\n            shuffle_seed = args.shuffle_seed\n\n    def _max_token_fn(current_idx, current_batch_size, tokens_sofar,\n                      data_source):\n        return max(tokens_sofar,\n                   len(data_source[current_idx][0]) + 1,\n                   len(data_source[current_idx][1]) + 1)\n\n    def _key(size_so_far, minibatch_len):\n        return size_so_far * minibatch_len\n\n    data_loaders = [(None)] * 2\n    for i, dataset in enumerate(datasets):\n        m = dataset.mode\n        dataset = dataset.filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n        sampler = SamplerHelper(dataset)\n\n        src_key = (lambda x, data_source: len(data_source[x][0]) + 1)\n        if args.sort_type == SortType.GLOBAL:\n            buffer_size = -1\n            trg_key = (lambda x, data_source: len(data_source[x][1]) + 1)\n            # Sort twice\n            sampler = sampler.sort(\n                key=trg_key, buffer_size=buffer_size).sort(\n                    key=src_key, buffer_size=buffer_size)\n        else:\n            if args.shuffle:\n                sampler = sampler.shuffle(seed=shuffle_seed)\n            if args.sort_type == SortType.POOL:\n                buffer_size = args.pool_size\n                sampler = sampler.sort(key=src_key, buffer_size=buffer_size)\n\n        batch_sampler = sampler.batch(\n            batch_size=args.batch_size,\n            drop_last=False,\n            batch_size_fn=_max_token_fn,\n            key=_key)\n\n        if m == \"train\":\n            batch_sampler = batch_sampler.shard()\n\n        if args.shuffle_batch:\n            batch_sampler.shuffle(seed=shuffle_seed)\n\n        data_loader = DataLoader(\n            dataset=dataset,\n            batch_sampler=batch_sampler,\n            collate_fn=partial(\n                prepare_train_input,\n                bos_idx=args.bos_idx,\n                eos_idx=args.eos_idx,\n                pad_idx=args.bos_idx),\n            num_workers=0,\n            return_list=True)\n        data_loaders[i] = (data_loader)\n    return data_loaders\n\n\ndef create_infer_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    dataset = WMT14ende.get_datasets(\n        mode=\"test\", transform_func=transform_func).filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n\n    batch_sampler = SamplerHelper(dataset).batch(\n        batch_size=args.infer_batch_size, drop_last=False)\n\n    data_loader = DataLoader(\n        dataset=dataset,\n        batch_sampler=batch_sampler,\n        collate_fn=partial(\n            prepare_infer_input,\n            bos_idx=args.bos_idx,\n            eos_idx=args.eos_idx,\n            pad_idx=args.bos_idx),\n        num_workers=0,\n        return_list=True)\n    return data_loader, trg_vocab.to_tokens\n\n\ndef prepare_train_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by training into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n    trg_word = word_pad([[bos_idx] + inst[1] for inst in insts])\n    lbl_word = np.expand_dims(\n        word_pad([inst[1] + [eos_idx] for inst in insts]), axis=2)\n\n    data_inputs = [src_word, trg_word, lbl_word]\n\n    return data_inputs\n\n\ndef prepare_infer_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by beam search decoder into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n\n    return [src_word, ]\n\n\nclass SortType(object):\n    GLOBAL = 'global'\n    POOL = 'pool'\n    NONE = \"none\"\n", "path": "PaddleNLP/benchmark/transformer/reader.py"}, {"content": "# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport sys\nimport os\nimport io\nimport itertools\nfrom functools import partial\n\nimport numpy as np\nfrom paddle.io import BatchSampler, DataLoader, Dataset\nfrom paddlenlp.data import Pad\nfrom paddlenlp.datasets import WMT14ende\nfrom paddlenlp.data.sampler import SamplerHelper\n\n\ndef min_max_filer(data, max_len, min_len=0):\n    # 1 for special tokens.\n    data_min_len = min(len(data[0]), len(data[1])) + 1\n    data_max_len = max(len(data[0]), len(data[1])) + 1\n    return (data_min_len >= min_len) and (data_max_len <= max_len)\n\n\ndef create_data_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    datasets = [\n        WMT14ende.get_datasets(\n            mode=m, transform_func=transform_func) for m in [\"train\", \"dev\"]\n    ]\n\n    if args.shuffle or args.shuffle_batch:\n        if args.shuffle_seed == \"None\" or args.shuffle_seed is None:\n            shuffle_seed = 0\n        else:\n            shuffle_seed = args.shuffle_seed\n\n    def _max_token_fn(current_idx, current_batch_size, tokens_sofar,\n                      data_source):\n        return max(tokens_sofar,\n                   len(data_source[current_idx][0]) + 1,\n                   len(data_source[current_idx][1]) + 1)\n\n    def _key(size_so_far, minibatch_len):\n        return size_so_far * minibatch_len\n\n    data_loaders = [(None)] * 2\n    for i, dataset in enumerate(datasets):\n        m = dataset.mode\n        dataset = dataset.filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n        sampler = SamplerHelper(dataset)\n\n        src_key = (lambda x, data_source: len(data_source[x][0]) + 1)\n        if args.sort_type == SortType.GLOBAL:\n            buffer_size = -1\n            trg_key = (lambda x, data_source: len(data_source[x][1]) + 1)\n            # Sort twice\n            sampler = sampler.sort(\n                key=trg_key, buffer_size=buffer_size).sort(\n                    key=src_key, buffer_size=buffer_size)\n        else:\n            if args.shuffle:\n                sampler = sampler.shuffle(seed=shuffle_seed)\n            if args.sort_type == SortType.POOL:\n                buffer_size = args.pool_size\n                sampler = sampler.sort(key=src_key, buffer_size=buffer_size)\n\n        batch_sampler = sampler.batch(\n            batch_size=args.batch_size,\n            drop_last=False,\n            batch_size_fn=_max_token_fn,\n            key=_key)\n\n        if m == \"train\":\n            batch_sampler = batch_sampler.shard()\n\n        if args.shuffle_batch:\n            batch_sampler.shuffle(seed=shuffle_seed)\n\n        data_loader = DataLoader(\n            dataset=dataset,\n            batch_sampler=batch_sampler,\n            collate_fn=partial(\n                prepare_train_input,\n                bos_idx=args.bos_idx,\n                eos_idx=args.eos_idx,\n                pad_idx=args.bos_idx),\n            num_workers=0,\n            return_list=True)\n        data_loaders[i] = (data_loader)\n    return data_loaders\n\n\ndef create_infer_loader(args):\n    root = None if args.root == \"None\" else args.root\n    (src_vocab, trg_vocab) = WMT14ende.get_vocab(root=root)\n    args.src_vocab_size, args.trg_vocab_size = len(src_vocab), len(trg_vocab)\n    transform_func = WMT14ende.get_default_transform_func(root=root)\n    dataset = WMT14ende.get_datasets(\n        mode=\"test\", transform_func=transform_func).filter(\n            partial(\n                min_max_filer, max_len=args.max_length))\n\n    batch_sampler = SamplerHelper(dataset).batch(\n        batch_size=args.infer_batch_size, drop_last=False)\n\n    data_loader = DataLoader(\n        dataset=dataset,\n        batch_sampler=batch_sampler,\n        collate_fn=partial(\n            prepare_infer_input,\n            bos_idx=args.bos_idx,\n            eos_idx=args.eos_idx,\n            pad_idx=args.bos_idx),\n        num_workers=0,\n        return_list=True)\n    return data_loader, trg_vocab.to_tokens\n\n\ndef prepare_train_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by training into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n    trg_word = word_pad([[bos_idx] + inst[1] for inst in insts])\n    lbl_word = np.expand_dims(\n        word_pad([inst[1] + [eos_idx] for inst in insts]), axis=2)\n\n    data_inputs = [src_word, trg_word, lbl_word]\n\n    return data_inputs\n\n\ndef prepare_infer_input(insts, bos_idx, eos_idx, pad_idx):\n    \"\"\"\n    Put all padded data needed by beam search decoder into a list.\n    \"\"\"\n    word_pad = Pad(pad_idx)\n    src_word = word_pad([inst[0] + [eos_idx] for inst in insts])\n\n    return [src_word, ]\n\n\nclass SortType(object):\n    GLOBAL = 'global'\n    POOL = 'pool'\n    NONE = \"none\"\n", "path": "PaddleNLP/examples/machine_translation/transformer/reader.py"}]}