We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BPTTIterator: Result of Slicing is an empty tensor
I can't iterate through all the batches in the iterator returned by BPTTITerator.
For example, this will give an error: batches = [batch for batch in train_iter]
On the last batch, I'll get error: Result of Slicing is an empty tensor

How should I fix it?

Thanks!
BPTTIterator: Result of Slicing is an empty tensor
I can't iterate through all the batches in the iterator returned by BPTTITerator.
For example, this will give an error: batches = [batch for batch in train_iter]
On the last batch, I'll get error: Result of Slicing is an empty tensor

How should I fix it?

Thanks!

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/data/iterator.py`
Content:
```
1 from __future__ import division
2 
3 import math
4 import random
5 from contextlib import contextmanager
6 from copy import deepcopy
7 
8 from .batch import Batch
9 from .dataset import Dataset
10 
11 
12 class RandomShuffler(object):
13     """Use random functions while keeping track of the random state to make it
14     reproducible and deterministic."""
15 
16     def __init__(self, random_state=None):
17         self._random_state = random_state
18         if self._random_state is None:
19             self._random_state = random.getstate()
20 
21     @contextmanager
22     def use_internal_state(self):
23         """Use a specific RNG state."""
24         old_state = random.getstate()
25         random.setstate(self._random_state)
26         yield
27         self._random_state = random.getstate()
28         random.setstate(old_state)
29 
30     @property
31     def random_state(self):
32         return deepcopy(self._random_state)
33 
34     @random_state.setter
35     def random_state(self, s):
36         self._random_state = s
37 
38     def __call__(self, data):
39         """Shuffle and return a new list."""
40         with self.use_internal_state():
41             return random.sample(data, len(data))
42 
43 
44 class Iterator(object):
45     """Defines an iterator that loads batches of data from a Dataset.
46 
47     Attributes:
48         dataset: The Dataset object to load Examples from.
49         batch_size: Batch size.
50         batch_size_fn: Function of three arguments (new example to add, current
51             count of examples in the batch, and current effective batch size)
52             that returns the new effective batch size resulting from adding
53             that example to a batch. This is useful for dynamic batching, where
54             this function would add to the current effective batch size the
55             number of tokens in the new example.
56         sort_key: A key to use for sorting examples in order to batch together
57             examples with similar lengths and minimize padding. The sort_key
58             provided to the Iterator constructor overrides the sort_key
59             attribute of the Dataset, or defers to it if None.
60         train: Whether the iterator represents a train set.
61         repeat: Whether to repeat the iterator for multiple epochs.
62         shuffle: Whether to shuffle examples between epochs.
63         sort: Whether to sort examples according to self.sort_key.
64             Note that repeat, shuffle, and sort default to train, train, and
65             (not train).
66         sort_within_batch: Whether to sort (in descending order according to
67             self.sort_key) within each batch. If None, defaults to self.sort.
68             If self.sort is True and this is False, the batch is left in the
69             original (ascending) sorted order.
70         device: Device to create batches on. Use -1 for CPU and None for the
71             currently active GPU device.
72     """
73 
74     def __init__(self, dataset, batch_size, sort_key=None, device=None,
75                  batch_size_fn=lambda new, count, sofar: count, train=True,
76                  repeat=None, shuffle=None, sort=None,
77                  sort_within_batch=None):
78         self.batch_size, self.train, self.dataset = batch_size, train, dataset
79         self.batch_size_fn = batch_size_fn
80         self.iterations = 0
81         self.repeat = train if repeat is None else repeat
82         self.shuffle = train if shuffle is None else shuffle
83         self.sort = not train if sort is None else sort
84         if sort_within_batch is None:
85             self.sort_within_batch = self.sort
86         else:
87             self.sort_within_batch = sort_within_batch
88         if sort_key is None:
89             self.sort_key = dataset.sort_key
90         else:
91             self.sort_key = sort_key
92         self.device = device
93 
94         self.random_shuffler = RandomShuffler()
95 
96         # For state loading/saving only
97         self._iterations_this_epoch = 0
98         self._random_state_this_epoch = None
99         self._restored_from_state = False
100 
101     @classmethod
102     def splits(cls, datasets, batch_sizes=None, **kwargs):
103         """Create Iterator objects for multiple splits of a dataset.
104 
105         Arguments:
106             datasets: Tuple of Dataset objects corresponding to the splits. The
107                 first such object should be the train set.
108             batch_sizes: Tuple of batch sizes to use for the different splits,
109                 or None to use the same batch_size for all splits.
110             Remaining keyword arguments: Passed to the constructor of the
111                 iterator class being used.
112         """
113         if batch_sizes is None:
114             batch_sizes = [kwargs.pop('batch_size')] * len(datasets)
115         ret = []
116         for i in range(len(datasets)):
117             train = i == 0
118             ret.append(cls(
119                 datasets[i], batch_size=batch_sizes[i], train=train, **kwargs))
120         return tuple(ret)
121 
122     def data(self):
123         """Return the examples in the dataset in order, sorted, or shuffled."""
124         if self.sort:
125             xs = sorted(self.dataset, key=self.sort_key)
126         elif self.shuffle:
127             xs = [self.dataset[i] for i in self.random_shuffler(range(len(self.dataset)))]
128         else:
129             xs = self.dataset
130         return xs
131 
132     def init_epoch(self):
133         """Set up the batch generator for a new epoch."""
134 
135         if self._restored_from_state:
136             self.random_shuffler.random_state = self._random_state_this_epoch
137         else:
138             self._random_state_this_epoch = self.random_shuffler.random_state
139 
140         self.create_batches()
141 
142         if self._restored_from_state:
143             self._restored_from_state = False
144         else:
145             self._iterations_this_epoch = 0
146 
147         if not self.repeat:
148             self.iterations = 0
149 
150     def create_batches(self):
151         self.batches = batch(self.data(), self.batch_size, self.batch_size_fn)
152 
153     @property
154     def epoch(self):
155         return self.iterations / len(self)
156 
157     def __len__(self):
158         return math.ceil(len(self.dataset) / self.batch_size)
159 
160     def __iter__(self):
161         while True:
162             self.init_epoch()
163             for idx, minibatch in enumerate(self.batches):
164                 # fast-forward if loaded from state
165                 if self._iterations_this_epoch > idx:
166                     continue
167                 self.iterations += 1
168                 self._iterations_this_epoch += 1
169                 if self.sort_within_batch:
170                     # NOTE: `rnn.pack_padded_sequence` requires that a minibatch
171                     # be sorted by decreasing order, which requires reversing
172                     # relative to typical sort keys
173                     if self.sort:
174                         minibatch.reverse()
175                     else:
176                         minibatch.sort(key=self.sort_key, reverse=True)
177                 yield Batch(minibatch, self.dataset, self.device,
178                             self.train)
179             if not self.repeat:
180                 raise StopIteration
181 
182     def state_dict(self):
183         return {
184             "iterations": self.iterations,
185             "iterations_this_epoch": self._iterations_this_epoch,
186             "random_state_this_epoch": self._random_state_this_epoch}
187 
188     def load_state_dict(self, state_dict):
189         self.iterations = state_dict["iterations"]
190         self._iterations_this_epoch = state_dict["iterations_this_epoch"]
191         self._random_state_this_epoch = state_dict["random_state_this_epoch"]
192         self._restored_from_state = True
193 
194 
195 class BPTTIterator(Iterator):
196     """Defines an iterator for language modeling tasks that use BPTT.
197 
198     Provides contiguous streams of examples together with targets that are
199     one timestep further forward, for language modeling training with
200     backpropagation through time (BPTT). Expects a Dataset with a single
201     example and a single field called 'text' and produces Batches with text and
202     target attributes.
203 
204     Attributes:
205         dataset: The Dataset object to load Examples from.
206         batch_size: Batch size.
207         bptt_len: Length of sequences for backpropagation through time.
208         sort_key: A key to use for sorting examples in order to batch together
209             examples with similar lengths and minimize padding. The sort_key
210             provided to the Iterator constructor overrides the sort_key
211             attribute of the Dataset, or defers to it if None.
212         train: Whether the iterator represents a train set.
213         repeat: Whether to repeat the iterator for multiple epochs.
214         shuffle: Whether to shuffle examples between epochs.
215         sort: Whether to sort examples according to self.sort_key.
216             Note that repeat, shuffle, and sort default to train, train, and
217             (not train).
218         device: Device to create batches on. Use -1 for CPU and None for the
219             currently active GPU device.
220     """
221 
222     def __init__(self, dataset, batch_size, bptt_len, **kwargs):
223         self.bptt_len = bptt_len
224         super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)
225 
226     def __len__(self):
227         return math.ceil(len(self.dataset[0].text) /
228                          (self.batch_size * self.bptt_len))
229 
230     def __iter__(self):
231         text = self.dataset[0].text
232         TEXT = self.dataset.fields['text']
233         TEXT.eos_token = None
234         text = text + ([TEXT.pad_token] * int(math.ceil(len(text) / self.batch_size) *
235                                               self.batch_size - len(text)))
236         data = TEXT.numericalize(
237             [text], device=self.device, train=self.train)
238         data = data.view(self.batch_size, -1).t().contiguous()
239         dataset = Dataset(examples=self.dataset.examples, fields=[
240             ('text', TEXT), ('target', TEXT)])
241         while True:
242             for i in range(0, len(self) * self.bptt_len, self.bptt_len):
243                 seq_len = min(self.bptt_len, len(data) - 1 - i)
244                 yield Batch.fromvars(
245                     dataset, self.batch_size, train=self.train,
246                     text=data[i:i + seq_len],
247                     target=data[i + 1:i + 1 + seq_len])
248             if not self.repeat:
249                 raise StopIteration
250 
251 
252 class BucketIterator(Iterator):
253     """Defines an iterator that batches examples of similar lengths together.
254 
255     Minimizes amount of padding needed while producing freshly shuffled
256     batches for each new epoch. See pool for the bucketing procedure used.
257     """
258 
259     def create_batches(self):
260         if self.sort:
261             self.batches = batch(self.data(), self.batch_size,
262                                  self.batch_size_fn)
263         else:
264             self.batches = pool(self.data(), self.batch_size,
265                                 self.sort_key, self.batch_size_fn,
266                                 random_shuffler=self.random_shuffler)
267 
268 
269 def batch(data, batch_size, batch_size_fn=lambda new, count, sofar: count):
270     """Yield elements from data in chunks of batch_size."""
271     minibatch, size_so_far = [], 0
272     for ex in data:
273         minibatch.append(ex)
274         size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
275         if size_so_far == batch_size:
276             yield minibatch
277             minibatch, size_so_far = [], 0
278         elif size_so_far > batch_size:
279             yield minibatch[:-1]
280             minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
281     if minibatch:
282         yield minibatch
283 
284 
285 def pool(data, batch_size, key, batch_size_fn=lambda new, count, sofar: count,
286          random_shuffler=None):
287     """Sort within buckets, then batch, then shuffle batches.
288 
289     Partitions data into chunks of size 100*batch_size, sorts examples within
290     each chunk using sort_key, then batch these examples and shuffle the
291     batches.
292     """
293     if random_shuffler is None:
294         random_shuffler = random.shuffle
295     for p in batch(data, batch_size * 100, batch_size_fn):
296         p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn)
297         for b in random_shuffler(list(p_batch)):
298             yield b
299
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/data/iterator.py b/torchtext/data/iterator.py
--- a/torchtext/data/iterator.py
+++ b/torchtext/data/iterator.py
@@ -224,7 +224,7 @@
         super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)
 
     def __len__(self):
-        return math.ceil(len(self.dataset[0].text) /
+        return math.ceil((len(self.dataset[0].text) - 1) /
                          (self.batch_size * self.bptt_len))
 
     def __iter__(self):
@@ -240,7 +240,7 @@
             ('text', TEXT), ('target', TEXT)])
         while True:
             for i in range(0, len(self) * self.bptt_len, self.bptt_len):
-                seq_len = min(self.bptt_len, len(data) - 1 - i)
+                seq_len = min(self.bptt_len, len(data) - i - 1)
                 yield Batch.fromvars(
                     dataset, self.batch_size, train=self.train,
                     text=data[i:i + seq_len],

{"golden_diff": "diff --git a/torchtext/data/iterator.py b/torchtext/data/iterator.py\n--- a/torchtext/data/iterator.py\n+++ b/torchtext/data/iterator.py\n@@ -224,7 +224,7 @@\n         super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)\n \n     def __len__(self):\n-        return math.ceil(len(self.dataset[0].text) /\n+        return math.ceil((len(self.dataset[0].text) - 1) /\n                          (self.batch_size * self.bptt_len))\n \n     def __iter__(self):\n@@ -240,7 +240,7 @@\n             ('text', TEXT), ('target', TEXT)])\n         while True:\n             for i in range(0, len(self) * self.bptt_len, self.bptt_len):\n-                seq_len = min(self.bptt_len, len(data) - 1 - i)\n+                seq_len = min(self.bptt_len, len(data) - i - 1)\n                 yield Batch.fromvars(\n                     dataset, self.batch_size, train=self.train,\n                     text=data[i:i + seq_len],\n", "issue": "BPTTIterator: Result of Slicing is an empty tensor\nI can't iterate through all the batches in the iterator returned by BPTTITerator.\r\nFor example, this will give an error: batches = [batch for batch in train_iter]\r\nOn the last batch, I'll get error: Result of Slicing is an empty tensor\r\n\r\nHow should I fix it?\r\n\r\nThanks!\nBPTTIterator: Result of Slicing is an empty tensor\nI can't iterate through all the batches in the iterator returned by BPTTITerator.\r\nFor example, this will give an error: batches = [batch for batch in train_iter]\r\nOn the last batch, I'll get error: Result of Slicing is an empty tensor\r\n\r\nHow should I fix it?\r\n\r\nThanks!\n", "before_files": [{"content": "from __future__ import division\n\nimport math\nimport random\nfrom contextlib import contextmanager\nfrom copy import deepcopy\n\nfrom .batch import Batch\nfrom .dataset import Dataset\n\n\nclass RandomShuffler(object):\n    \"\"\"Use random functions while keeping track of the random state to make it\n    reproducible and deterministic.\"\"\"\n\n    def __init__(self, random_state=None):\n        self._random_state = random_state\n        if self._random_state is None:\n            self._random_state = random.getstate()\n\n    @contextmanager\n    def use_internal_state(self):\n        \"\"\"Use a specific RNG state.\"\"\"\n        old_state = random.getstate()\n        random.setstate(self._random_state)\n        yield\n        self._random_state = random.getstate()\n        random.setstate(old_state)\n\n    @property\n    def random_state(self):\n        return deepcopy(self._random_state)\n\n    @random_state.setter\n    def random_state(self, s):\n        self._random_state = s\n\n    def __call__(self, data):\n        \"\"\"Shuffle and return a new list.\"\"\"\n        with self.use_internal_state():\n            return random.sample(data, len(data))\n\n\nclass Iterator(object):\n    \"\"\"Defines an iterator that loads batches of data from a Dataset.\n\n    Attributes:\n        dataset: The Dataset object to load Examples from.\n        batch_size: Batch size.\n        batch_size_fn: Function of three arguments (new example to add, current\n            count of examples in the batch, and current effective batch size)\n            that returns the new effective batch size resulting from adding\n            that example to a batch. This is useful for dynamic batching, where\n            this function would add to the current effective batch size the\n            number of tokens in the new example.\n        sort_key: A key to use for sorting examples in order to batch together\n            examples with similar lengths and minimize padding. The sort_key\n            provided to the Iterator constructor overrides the sort_key\n            attribute of the Dataset, or defers to it if None.\n        train: Whether the iterator represents a train set.\n        repeat: Whether to repeat the iterator for multiple epochs.\n        shuffle: Whether to shuffle examples between epochs.\n        sort: Whether to sort examples according to self.sort_key.\n            Note that repeat, shuffle, and sort default to train, train, and\n            (not train).\n        sort_within_batch: Whether to sort (in descending order according to\n            self.sort_key) within each batch. If None, defaults to self.sort.\n            If self.sort is True and this is False, the batch is left in the\n            original (ascending) sorted order.\n        device: Device to create batches on. Use -1 for CPU and None for the\n            currently active GPU device.\n    \"\"\"\n\n    def __init__(self, dataset, batch_size, sort_key=None, device=None,\n                 batch_size_fn=lambda new, count, sofar: count, train=True,\n                 repeat=None, shuffle=None, sort=None,\n                 sort_within_batch=None):\n        self.batch_size, self.train, self.dataset = batch_size, train, dataset\n        self.batch_size_fn = batch_size_fn\n        self.iterations = 0\n        self.repeat = train if repeat is None else repeat\n        self.shuffle = train if shuffle is None else shuffle\n        self.sort = not train if sort is None else sort\n        if sort_within_batch is None:\n            self.sort_within_batch = self.sort\n        else:\n            self.sort_within_batch = sort_within_batch\n        if sort_key is None:\n            self.sort_key = dataset.sort_key\n        else:\n            self.sort_key = sort_key\n        self.device = device\n\n        self.random_shuffler = RandomShuffler()\n\n        # For state loading/saving only\n        self._iterations_this_epoch = 0\n        self._random_state_this_epoch = None\n        self._restored_from_state = False\n\n    @classmethod\n    def splits(cls, datasets, batch_sizes=None, **kwargs):\n        \"\"\"Create Iterator objects for multiple splits of a dataset.\n\n        Arguments:\n            datasets: Tuple of Dataset objects corresponding to the splits. The\n                first such object should be the train set.\n            batch_sizes: Tuple of batch sizes to use for the different splits,\n                or None to use the same batch_size for all splits.\n            Remaining keyword arguments: Passed to the constructor of the\n                iterator class being used.\n        \"\"\"\n        if batch_sizes is None:\n            batch_sizes = [kwargs.pop('batch_size')] * len(datasets)\n        ret = []\n        for i in range(len(datasets)):\n            train = i == 0\n            ret.append(cls(\n                datasets[i], batch_size=batch_sizes[i], train=train, **kwargs))\n        return tuple(ret)\n\n    def data(self):\n        \"\"\"Return the examples in the dataset in order, sorted, or shuffled.\"\"\"\n        if self.sort:\n            xs = sorted(self.dataset, key=self.sort_key)\n        elif self.shuffle:\n            xs = [self.dataset[i] for i in self.random_shuffler(range(len(self.dataset)))]\n        else:\n            xs = self.dataset\n        return xs\n\n    def init_epoch(self):\n        \"\"\"Set up the batch generator for a new epoch.\"\"\"\n\n        if self._restored_from_state:\n            self.random_shuffler.random_state = self._random_state_this_epoch\n        else:\n            self._random_state_this_epoch = self.random_shuffler.random_state\n\n        self.create_batches()\n\n        if self._restored_from_state:\n            self._restored_from_state = False\n        else:\n            self._iterations_this_epoch = 0\n\n        if not self.repeat:\n            self.iterations = 0\n\n    def create_batches(self):\n        self.batches = batch(self.data(), self.batch_size, self.batch_size_fn)\n\n    @property\n    def epoch(self):\n        return self.iterations / len(self)\n\n    def __len__(self):\n        return math.ceil(len(self.dataset) / self.batch_size)\n\n    def __iter__(self):\n        while True:\n            self.init_epoch()\n            for idx, minibatch in enumerate(self.batches):\n                # fast-forward if loaded from state\n                if self._iterations_this_epoch > idx:\n                    continue\n                self.iterations += 1\n                self._iterations_this_epoch += 1\n                if self.sort_within_batch:\n                    # NOTE: `rnn.pack_padded_sequence` requires that a minibatch\n                    # be sorted by decreasing order, which requires reversing\n                    # relative to typical sort keys\n                    if self.sort:\n                        minibatch.reverse()\n                    else:\n                        minibatch.sort(key=self.sort_key, reverse=True)\n                yield Batch(minibatch, self.dataset, self.device,\n                            self.train)\n            if not self.repeat:\n                raise StopIteration\n\n    def state_dict(self):\n        return {\n            \"iterations\": self.iterations,\n            \"iterations_this_epoch\": self._iterations_this_epoch,\n            \"random_state_this_epoch\": self._random_state_this_epoch}\n\n    def load_state_dict(self, state_dict):\n        self.iterations = state_dict[\"iterations\"]\n        self._iterations_this_epoch = state_dict[\"iterations_this_epoch\"]\n        self._random_state_this_epoch = state_dict[\"random_state_this_epoch\"]\n        self._restored_from_state = True\n\n\nclass BPTTIterator(Iterator):\n    \"\"\"Defines an iterator for language modeling tasks that use BPTT.\n\n    Provides contiguous streams of examples together with targets that are\n    one timestep further forward, for language modeling training with\n    backpropagation through time (BPTT). Expects a Dataset with a single\n    example and a single field called 'text' and produces Batches with text and\n    target attributes.\n\n    Attributes:\n        dataset: The Dataset object to load Examples from.\n        batch_size: Batch size.\n        bptt_len: Length of sequences for backpropagation through time.\n        sort_key: A key to use for sorting examples in order to batch together\n            examples with similar lengths and minimize padding. The sort_key\n            provided to the Iterator constructor overrides the sort_key\n            attribute of the Dataset, or defers to it if None.\n        train: Whether the iterator represents a train set.\n        repeat: Whether to repeat the iterator for multiple epochs.\n        shuffle: Whether to shuffle examples between epochs.\n        sort: Whether to sort examples according to self.sort_key.\n            Note that repeat, shuffle, and sort default to train, train, and\n            (not train).\n        device: Device to create batches on. Use -1 for CPU and None for the\n            currently active GPU device.\n    \"\"\"\n\n    def __init__(self, dataset, batch_size, bptt_len, **kwargs):\n        self.bptt_len = bptt_len\n        super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)\n\n    def __len__(self):\n        return math.ceil(len(self.dataset[0].text) /\n                         (self.batch_size * self.bptt_len))\n\n    def __iter__(self):\n        text = self.dataset[0].text\n        TEXT = self.dataset.fields['text']\n        TEXT.eos_token = None\n        text = text + ([TEXT.pad_token] * int(math.ceil(len(text) / self.batch_size) *\n                                              self.batch_size - len(text)))\n        data = TEXT.numericalize(\n            [text], device=self.device, train=self.train)\n        data = data.view(self.batch_size, -1).t().contiguous()\n        dataset = Dataset(examples=self.dataset.examples, fields=[\n            ('text', TEXT), ('target', TEXT)])\n        while True:\n            for i in range(0, len(self) * self.bptt_len, self.bptt_len):\n                seq_len = min(self.bptt_len, len(data) - 1 - i)\n                yield Batch.fromvars(\n                    dataset, self.batch_size, train=self.train,\n                    text=data[i:i + seq_len],\n                    target=data[i + 1:i + 1 + seq_len])\n            if not self.repeat:\n                raise StopIteration\n\n\nclass BucketIterator(Iterator):\n    \"\"\"Defines an iterator that batches examples of similar lengths together.\n\n    Minimizes amount of padding needed while producing freshly shuffled\n    batches for each new epoch. See pool for the bucketing procedure used.\n    \"\"\"\n\n    def create_batches(self):\n        if self.sort:\n            self.batches = batch(self.data(), self.batch_size,\n                                 self.batch_size_fn)\n        else:\n            self.batches = pool(self.data(), self.batch_size,\n                                self.sort_key, self.batch_size_fn,\n                                random_shuffler=self.random_shuffler)\n\n\ndef batch(data, batch_size, batch_size_fn=lambda new, count, sofar: count):\n    \"\"\"Yield elements from data in chunks of batch_size.\"\"\"\n    minibatch, size_so_far = [], 0\n    for ex in data:\n        minibatch.append(ex)\n        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)\n        if size_so_far == batch_size:\n            yield minibatch\n            minibatch, size_so_far = [], 0\n        elif size_so_far > batch_size:\n            yield minibatch[:-1]\n            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)\n    if minibatch:\n        yield minibatch\n\n\ndef pool(data, batch_size, key, batch_size_fn=lambda new, count, sofar: count,\n         random_shuffler=None):\n    \"\"\"Sort within buckets, then batch, then shuffle batches.\n\n    Partitions data into chunks of size 100*batch_size, sorts examples within\n    each chunk using sort_key, then batch these examples and shuffle the\n    batches.\n    \"\"\"\n    if random_shuffler is None:\n        random_shuffler = random.shuffle\n    for p in batch(data, batch_size * 100, batch_size_fn):\n        p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn)\n        for b in random_shuffler(list(p_batch)):\n            yield b\n", "path": "torchtext/data/iterator.py"}], "after_files": [{"content": "from __future__ import division\n\nimport math\nimport random\nfrom contextlib import contextmanager\nfrom copy import deepcopy\n\nfrom .batch import Batch\nfrom .dataset import Dataset\n\n\nclass RandomShuffler(object):\n    \"\"\"Use random functions while keeping track of the random state to make it\n    reproducible and deterministic.\"\"\"\n\n    def __init__(self, random_state=None):\n        self._random_state = random_state\n        if self._random_state is None:\n            self._random_state = random.getstate()\n\n    @contextmanager\n    def use_internal_state(self):\n        \"\"\"Use a specific RNG state.\"\"\"\n        old_state = random.getstate()\n        random.setstate(self._random_state)\n        yield\n        self._random_state = random.getstate()\n        random.setstate(old_state)\n\n    @property\n    def random_state(self):\n        return deepcopy(self._random_state)\n\n    @random_state.setter\n    def random_state(self, s):\n        self._random_state = s\n\n    def __call__(self, data):\n        \"\"\"Shuffle and return a new list.\"\"\"\n        with self.use_internal_state():\n            return random.sample(data, len(data))\n\n\nclass Iterator(object):\n    \"\"\"Defines an iterator that loads batches of data from a Dataset.\n\n    Attributes:\n        dataset: The Dataset object to load Examples from.\n        batch_size: Batch size.\n        batch_size_fn: Function of three arguments (new example to add, current\n            count of examples in the batch, and current effective batch size)\n            that returns the new effective batch size resulting from adding\n            that example to a batch. This is useful for dynamic batching, where\n            this function would add to the current effective batch size the\n            number of tokens in the new example.\n        sort_key: A key to use for sorting examples in order to batch together\n            examples with similar lengths and minimize padding. The sort_key\n            provided to the Iterator constructor overrides the sort_key\n            attribute of the Dataset, or defers to it if None.\n        train: Whether the iterator represents a train set.\n        repeat: Whether to repeat the iterator for multiple epochs.\n        shuffle: Whether to shuffle examples between epochs.\n        sort: Whether to sort examples according to self.sort_key.\n            Note that repeat, shuffle, and sort default to train, train, and\n            (not train).\n        sort_within_batch: Whether to sort (in descending order according to\n            self.sort_key) within each batch. If None, defaults to self.sort.\n            If self.sort is True and this is False, the batch is left in the\n            original (ascending) sorted order.\n        device: Device to create batches on. Use -1 for CPU and None for the\n            currently active GPU device.\n    \"\"\"\n\n    def __init__(self, dataset, batch_size, sort_key=None, device=None,\n                 batch_size_fn=lambda new, count, sofar: count, train=True,\n                 repeat=None, shuffle=None, sort=None,\n                 sort_within_batch=None):\n        self.batch_size, self.train, self.dataset = batch_size, train, dataset\n        self.batch_size_fn = batch_size_fn\n        self.iterations = 0\n        self.repeat = train if repeat is None else repeat\n        self.shuffle = train if shuffle is None else shuffle\n        self.sort = not train if sort is None else sort\n        if sort_within_batch is None:\n            self.sort_within_batch = self.sort\n        else:\n            self.sort_within_batch = sort_within_batch\n        if sort_key is None:\n            self.sort_key = dataset.sort_key\n        else:\n            self.sort_key = sort_key\n        self.device = device\n\n        self.random_shuffler = RandomShuffler()\n\n        # For state loading/saving only\n        self._iterations_this_epoch = 0\n        self._random_state_this_epoch = None\n        self._restored_from_state = False\n\n    @classmethod\n    def splits(cls, datasets, batch_sizes=None, **kwargs):\n        \"\"\"Create Iterator objects for multiple splits of a dataset.\n\n        Arguments:\n            datasets: Tuple of Dataset objects corresponding to the splits. The\n                first such object should be the train set.\n            batch_sizes: Tuple of batch sizes to use for the different splits,\n                or None to use the same batch_size for all splits.\n            Remaining keyword arguments: Passed to the constructor of the\n                iterator class being used.\n        \"\"\"\n        if batch_sizes is None:\n            batch_sizes = [kwargs.pop('batch_size')] * len(datasets)\n        ret = []\n        for i in range(len(datasets)):\n            train = i == 0\n            ret.append(cls(\n                datasets[i], batch_size=batch_sizes[i], train=train, **kwargs))\n        return tuple(ret)\n\n    def data(self):\n        \"\"\"Return the examples in the dataset in order, sorted, or shuffled.\"\"\"\n        if self.sort:\n            xs = sorted(self.dataset, key=self.sort_key)\n        elif self.shuffle:\n            xs = [self.dataset[i] for i in self.random_shuffler(range(len(self.dataset)))]\n        else:\n            xs = self.dataset\n        return xs\n\n    def init_epoch(self):\n        \"\"\"Set up the batch generator for a new epoch.\"\"\"\n\n        if self._restored_from_state:\n            self.random_shuffler.random_state = self._random_state_this_epoch\n        else:\n            self._random_state_this_epoch = self.random_shuffler.random_state\n\n        self.create_batches()\n\n        if self._restored_from_state:\n            self._restored_from_state = False\n        else:\n            self._iterations_this_epoch = 0\n\n        if not self.repeat:\n            self.iterations = 0\n\n    def create_batches(self):\n        self.batches = batch(self.data(), self.batch_size, self.batch_size_fn)\n\n    @property\n    def epoch(self):\n        return self.iterations / len(self)\n\n    def __len__(self):\n        return math.ceil(len(self.dataset) / self.batch_size)\n\n    def __iter__(self):\n        while True:\n            self.init_epoch()\n            for idx, minibatch in enumerate(self.batches):\n                # fast-forward if loaded from state\n                if self._iterations_this_epoch > idx:\n                    continue\n                self.iterations += 1\n                self._iterations_this_epoch += 1\n                if self.sort_within_batch:\n                    # NOTE: `rnn.pack_padded_sequence` requires that a minibatch\n                    # be sorted by decreasing order, which requires reversing\n                    # relative to typical sort keys\n                    if self.sort:\n                        minibatch.reverse()\n                    else:\n                        minibatch.sort(key=self.sort_key, reverse=True)\n                yield Batch(minibatch, self.dataset, self.device,\n                            self.train)\n            if not self.repeat:\n                raise StopIteration\n\n    def state_dict(self):\n        return {\n            \"iterations\": self.iterations,\n            \"iterations_this_epoch\": self._iterations_this_epoch,\n            \"random_state_this_epoch\": self._random_state_this_epoch}\n\n    def load_state_dict(self, state_dict):\n        self.iterations = state_dict[\"iterations\"]\n        self._iterations_this_epoch = state_dict[\"iterations_this_epoch\"]\n        self._random_state_this_epoch = state_dict[\"random_state_this_epoch\"]\n        self._restored_from_state = True\n\n\nclass BPTTIterator(Iterator):\n    \"\"\"Defines an iterator for language modeling tasks that use BPTT.\n\n    Provides contiguous streams of examples together with targets that are\n    one timestep further forward, for language modeling training with\n    backpropagation through time (BPTT). Expects a Dataset with a single\n    example and a single field called 'text' and produces Batches with text and\n    target attributes.\n\n    Attributes:\n        dataset: The Dataset object to load Examples from.\n        batch_size: Batch size.\n        bptt_len: Length of sequences for backpropagation through time.\n        sort_key: A key to use for sorting examples in order to batch together\n            examples with similar lengths and minimize padding. The sort_key\n            provided to the Iterator constructor overrides the sort_key\n            attribute of the Dataset, or defers to it if None.\n        train: Whether the iterator represents a train set.\n        repeat: Whether to repeat the iterator for multiple epochs.\n        shuffle: Whether to shuffle examples between epochs.\n        sort: Whether to sort examples according to self.sort_key.\n            Note that repeat, shuffle, and sort default to train, train, and\n            (not train).\n        device: Device to create batches on. Use -1 for CPU and None for the\n            currently active GPU device.\n    \"\"\"\n\n    def __init__(self, dataset, batch_size, bptt_len, **kwargs):\n        self.bptt_len = bptt_len\n        super(BPTTIterator, self).__init__(dataset, batch_size, **kwargs)\n\n    def __len__(self):\n        return math.ceil((len(self.dataset[0].text) - 1) /\n                         (self.batch_size * self.bptt_len))\n\n    def __iter__(self):\n        text = self.dataset[0].text\n        TEXT = self.dataset.fields['text']\n        TEXT.eos_token = None\n        text = text + ([TEXT.pad_token] * int(math.ceil(len(text) / self.batch_size) *\n                                              self.batch_size - len(text)))\n        data = TEXT.numericalize(\n            [text], device=self.device, train=self.train)\n        data = data.view(self.batch_size, -1).t().contiguous()\n        dataset = Dataset(examples=self.dataset.examples, fields=[\n            ('text', TEXT), ('target', TEXT)])\n        while True:\n            for i in range(0, len(self) * self.bptt_len, self.bptt_len):\n                seq_len = min(self.bptt_len, len(data) - i - 1)\n                yield Batch.fromvars(\n                    dataset, self.batch_size, train=self.train,\n                    text=data[i:i + seq_len],\n                    target=data[i + 1:i + 1 + seq_len])\n            if not self.repeat:\n                raise StopIteration\n\n\nclass BucketIterator(Iterator):\n    \"\"\"Defines an iterator that batches examples of similar lengths together.\n\n    Minimizes amount of padding needed while producing freshly shuffled\n    batches for each new epoch. See pool for the bucketing procedure used.\n    \"\"\"\n\n    def create_batches(self):\n        if self.sort:\n            self.batches = batch(self.data(), self.batch_size,\n                                 self.batch_size_fn)\n        else:\n            self.batches = pool(self.data(), self.batch_size,\n                                self.sort_key, self.batch_size_fn,\n                                random_shuffler=self.random_shuffler)\n\n\ndef batch(data, batch_size, batch_size_fn=lambda new, count, sofar: count):\n    \"\"\"Yield elements from data in chunks of batch_size.\"\"\"\n    minibatch, size_so_far = [], 0\n    for ex in data:\n        minibatch.append(ex)\n        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)\n        if size_so_far == batch_size:\n            yield minibatch\n            minibatch, size_so_far = [], 0\n        elif size_so_far > batch_size:\n            yield minibatch[:-1]\n            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)\n    if minibatch:\n        yield minibatch\n\n\ndef pool(data, batch_size, key, batch_size_fn=lambda new, count, sofar: count,\n         random_shuffler=None):\n    \"\"\"Sort within buckets, then batch, then shuffle batches.\n\n    Partitions data into chunks of size 100*batch_size, sorts examples within\n    each chunk using sort_key, then batch these examples and shuffle the\n    batches.\n    \"\"\"\n    if random_shuffler is None:\n        random_shuffler = random.shuffle\n    for p in batch(data, batch_size * 100, batch_size_fn):\n        p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn)\n        for b in random_shuffler(list(p_batch)):\n            yield b\n", "path": "torchtext/data/iterator.py"}]}