We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
dataset.sort_key not retained after dataset.split()
Hi I was trying out the new `split()` functionality to split a test set into test and validation set. When the `BucketIterator` is being set up from a newly split dataset the sorting fails because `dataset.sort_key` is None. Looking at the `split()`  function I see that a new instance of type `Dataset` is created in which `sort_key` is None by default:
```
def split():
...
if not stratified:
            train_data, test_data, val_data = rationed_split(self.examples, train_ratio,
                                                             test_ratio, val_ratio, rnd)
            return tuple(Dataset(d, self.fields)
                         for d in (train_data, val_data, test_data) if d)
```

I guess one way would be to explicitly copy the `sort_key` into the new instance. A more generic way could be to make a copy of the current instance and replace only the `examples` attribute but I'm not sure if that is really needed.

Thanks!

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/data/dataset.py`
Content:
```
1 import io
2 import os
3 import zipfile
4 import tarfile
5 from functools import partial
6 
7 import torch.utils.data
8 
9 from .utils import RandomShuffler
10 from .example import Example
11 from ..utils import download_from_url, unicode_csv_reader
12 
13 
14 class Dataset(torch.utils.data.Dataset):
15     """Defines a dataset composed of Examples along with its Fields.
16 
17     Attributes:
18         sort_key (callable): A key to use for sorting dataset examples for batching
19             together examples with similar lengths to minimize padding.
20         examples (list(Example)): The examples in this dataset.
21         fields (dict[str, Field]): Contains the name of each column or field, together
22             with the corresponding Field object. Two fields with the same Field object
23             will have a shared vocabulary.
24     """
25     sort_key = None
26 
27     def __init__(self, examples, fields, filter_pred=None):
28         """Create a dataset from a list of Examples and Fields.
29 
30         Arguments:
31             examples: List of Examples.
32             fields (List(tuple(str, Field))): The Fields to use in this tuple. The
33                 string is a field name, and the Field is the associated field.
34             filter_pred (callable or None): Use only examples for which
35                 filter_pred(example) is True, or use all examples if None.
36                 Default is None.
37         """
38         if filter_pred is not None:
39             make_list = isinstance(examples, list)
40             examples = filter(filter_pred, examples)
41             if make_list:
42                 examples = list(examples)
43         self.examples = examples
44         self.fields = dict(fields)
45         # Unpack field tuples
46         for n, f in list(self.fields.items()):
47             if isinstance(n, tuple):
48                 self.fields.update(zip(n, f))
49                 del self.fields[n]
50 
51     @classmethod
52     def splits(cls, path=None, root='.data', train=None, validation=None,
53                test=None, **kwargs):
54         """Create Dataset objects for multiple splits of a dataset.
55 
56         Arguments:
57             path (str): Common prefix of the splits' file paths, or None to use
58                 the result of cls.download(root).
59             root (str): Root dataset storage directory. Default is '.data'.
60             train (str): Suffix to add to path for the train set, or None for no
61                 train set. Default is None.
62             validation (str): Suffix to add to path for the validation set, or None
63                 for no validation set. Default is None.
64             test (str): Suffix to add to path for the test set, or None for no test
65                 set. Default is None.
66             Remaining keyword arguments: Passed to the constructor of the
67                 Dataset (sub)class being used.
68 
69         Returns:
70             Tuple[Dataset]: Datasets for train, validation, and
71                 test splits in that order, if provided.
72         """
73         if path is None:
74             path = cls.download(root)
75         train_data = None if train is None else cls(
76             os.path.join(path, train), **kwargs)
77         val_data = None if validation is None else cls(
78             os.path.join(path, validation), **kwargs)
79         test_data = None if test is None else cls(
80             os.path.join(path, test), **kwargs)
81         return tuple(d for d in (train_data, val_data, test_data)
82                      if d is not None)
83 
84     def split(self, split_ratio=0.7, stratified=False, strata_field='label',
85               random_state=None):
86         """Create train-test(-valid?) splits from the instance's examples.
87 
88         Arguments:
89             split_ratio (float or List of floats): a number [0, 1] denoting the amount
90                 of data to be used for the training split (rest is used for validation),
91                 or a list of numbers denoting the relative sizes of train, test and valid
92                 splits respectively. If the relative size for valid is missing, only the
93                 train-test split is returned. Default is 0.7 (for th train set).
94             stratified (bool): whether the sampling should be stratified.
95                 Default is False.
96             strata_field (str): name of the examples Field stratified over.
97                 Default is 'label' for the conventional label field.
98             random_state (int): the random seed used for shuffling.
99 
100         Returns:
101             Tuple[Dataset]: Datasets for train, validation, and
102                 test splits in that order, if the splits are provided.
103         """
104         train_ratio, test_ratio, val_ratio = check_split_ratio(split_ratio)
105 
106         # For the permutations
107         rnd = RandomShuffler(random_state)
108         if not stratified:
109             train_data, test_data, val_data = rationed_split(self.examples, train_ratio,
110                                                              test_ratio, val_ratio, rnd)
111             return tuple(Dataset(d, self.fields)
112                          for d in (train_data, val_data, test_data) if d)
113         else:
114             if strata_field not in self.fields:
115                 raise ValueError("Invalid field name for strata_field {}"
116                                  .format(strata_field))
117             strata = stratify(self.examples, strata_field)
118             train_data, test_data, val_data = [], [], []
119             for group in strata:
120                 # Stratify each group and add together the indices.
121                 group_train, group_test, group_val = rationed_split(group, train_ratio,
122                                                                     test_ratio, val_ratio,
123                                                                     rnd)
124                 train_data += group_train
125                 test_data += group_test
126                 val_data += group_val
127 
128             return tuple(Dataset(d, self.fields)
129                          for d in (train_data, val_data, test_data) if d)
130 
131     def __getitem__(self, i):
132         return self.examples[i]
133 
134     def __len__(self):
135         try:
136             return len(self.examples)
137         except TypeError:
138             return 2**32
139 
140     def __iter__(self):
141         for x in self.examples:
142             yield x
143 
144     def __getattr__(self, attr):
145         if attr in self.fields:
146             for x in self.examples:
147                 yield getattr(x, attr)
148 
149     @classmethod
150     def download(cls, root, check=None):
151         """Download and unzip an online archive (.zip, .gz, or .tgz).
152 
153         Arguments:
154             root (str): Folder to download data to.
155             check (str or None): Folder whose existence indicates
156                 that the dataset has already been downloaded, or
157                 None to check the existence of root/{cls.name}.
158 
159         Returns:
160             str: Path to extracted dataset.
161         """
162         path = os.path.join(root, cls.name)
163         check = path if check is None else check
164         if not os.path.isdir(check):
165             for url in cls.urls:
166                 if isinstance(url, tuple):
167                     url, filename = url
168                 else:
169                     filename = os.path.basename(url)
170                 zpath = os.path.join(path, filename)
171                 if not os.path.isfile(zpath):
172                     if not os.path.exists(os.path.dirname(zpath)):
173                         os.makedirs(os.path.dirname(zpath))
174                     print('downloading {}'.format(filename))
175                     download_from_url(url, zpath)
176                 ext = os.path.splitext(filename)[-1]
177                 if ext == '.zip':
178                     with zipfile.ZipFile(zpath, 'r') as zfile:
179                         print('extracting')
180                         zfile.extractall(path)
181                 elif ext in ['.gz', '.tgz']:
182                     with tarfile.open(zpath, 'r:gz') as tar:
183                         dirs = [member for member in tar.getmembers()]
184                         tar.extractall(path=path, members=dirs)
185         return os.path.join(path, cls.dirname)
186 
187 
188 class TabularDataset(Dataset):
189     """Defines a Dataset of columns stored in CSV, TSV, or JSON format."""
190 
191     def __init__(self, path, format, fields, skip_header=False, **kwargs):
192         """Create a TabularDataset given a path, file format, and field list.
193 
194         Arguments:
195             path (str): Path to the data file.
196             format (str): The format of the data file. One of "CSV", "TSV", or
197                 "JSON" (case-insensitive).
198             fields (list(tuple(str, Field)) or dict[str: tuple(str, Field)]:
199                 If using a list, the format must be CSV or TSV, and the values of the list
200                 should be tuples of (name, field).
201                 The fields should be in the same order as the columns in the CSV or TSV
202                 file, while tuples of (name, None) represent columns that will be ignored.
203 
204                 If using a dict, the keys should be a subset of the JSON keys or CSV/TSV
205                 columns, and the values should be tuples of (name, field).
206                 Keys not present in the input dictionary are ignored.
207                 This allows the user to rename columns from their JSON/CSV/TSV key names
208                 and also enables selecting a subset of columns to load.
209             skip_header (bool): Whether to skip the first line of the input file.
210         """
211         make_example = {
212             'json': Example.fromJSON, 'dict': Example.fromdict,
213             'tsv': Example.fromCSV, 'csv': Example.fromCSV}[format.lower()]
214 
215         with io.open(os.path.expanduser(path), encoding="utf8") as f:
216             if format == 'csv':
217                 reader = unicode_csv_reader(f)
218             elif format == 'tsv':
219                 reader = unicode_csv_reader(f, delimiter='\t')
220             else:
221                 reader = f
222 
223             if format in ['csv', 'tsv'] and isinstance(fields, dict):
224                 if skip_header:
225                     raise ValueError('When using a dict to specify fields with a {} file,'
226                                      'skip_header must be False and'
227                                      'the file must have a header.'.format(format))
228                 header = next(reader)
229                 field_to_index = {f: header.index(f) for f in fields.keys()}
230                 make_example = partial(make_example, field_to_index=field_to_index)
231 
232             if skip_header:
233                 next(reader)
234 
235             examples = [make_example(line, fields) for line in reader]
236 
237         if isinstance(fields, dict):
238             fields, field_dict = [], fields
239             for field in field_dict.values():
240                 if isinstance(field, list):
241                     fields.extend(field)
242                 else:
243                     fields.append(field)
244 
245         super(TabularDataset, self).__init__(examples, fields, **kwargs)
246 
247 
248 def check_split_ratio(split_ratio):
249     """Check that the split ratio argument is not malformed"""
250     valid_ratio = 0.
251     if isinstance(split_ratio, float):
252         # Only the train set relative ratio is provided
253         # Assert in bounds, validation size is zero
254         assert split_ratio > 0. and split_ratio < 1., (
255             "Split ratio {} not between 0 and 1".format(split_ratio))
256 
257         test_ratio = 1. - split_ratio
258         return (split_ratio, test_ratio, valid_ratio)
259     elif isinstance(split_ratio, list):
260         # A list of relative ratios is provided
261         length = len(split_ratio)
262         assert length == 2 or length == 3, (
263             "Length of split ratio list should be 2 or 3, got {}".format(split_ratio))
264 
265         # Normalize if necessary
266         ratio_sum = sum(split_ratio)
267         if not ratio_sum == 1.:
268             split_ratio = [float(ratio) / ratio_sum for ratio in split_ratio]
269 
270         if length == 2:
271             return tuple(split_ratio + [valid_ratio])
272         return tuple(split_ratio)
273     else:
274         raise ValueError('Split ratio must be float or a list, got {}'
275                          .format(type(split_ratio)))
276 
277 
278 def stratify(examples, strata_field):
279     # The field has to be hashable otherwise this doesn't work
280     # There's two iterations over the whole dataset here, which can be
281     # reduced to just one if a dedicated method for stratified splitting is used
282     unique_strata = set(getattr(example, strata_field) for example in examples)
283     strata_maps = {s: [] for s in unique_strata}
284     for example in examples:
285         strata_maps[getattr(example, strata_field)].append(example)
286     return list(strata_maps.values())
287 
288 
289 def rationed_split(examples, train_ratio, test_ratio, val_ratio, rnd):
290     # Create a random permutation of examples, then split them
291     # by ratio x length slices for each of the train/test/dev? splits
292     N = len(examples)
293     randperm = rnd(range(N))
294     train_len = int(round(train_ratio * N))
295 
296     # Due to possible rounding problems
297     if not val_ratio:
298         test_len = N - train_len
299     else:
300         test_len = int(round(test_ratio * N))
301 
302     indices = (randperm[:train_len],  # Train
303                randperm[train_len:train_len + test_len],  # Test
304                randperm[train_len + test_len:])  # Validation
305 
306     # There's a possibly empty list for the validation set
307     data = tuple([examples[i] for i in index] for index in indices)
308 
309     return data
310
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/data/dataset.py b/torchtext/data/dataset.py
--- a/torchtext/data/dataset.py
+++ b/torchtext/data/dataset.py
@@ -108,8 +108,6 @@
         if not stratified:
             train_data, test_data, val_data = rationed_split(self.examples, train_ratio,
                                                              test_ratio, val_ratio, rnd)
-            return tuple(Dataset(d, self.fields)
-                         for d in (train_data, val_data, test_data) if d)
         else:
             if strata_field not in self.fields:
                 raise ValueError("Invalid field name for strata_field {}"
@@ -125,8 +123,14 @@
                 test_data += group_test
                 val_data += group_val
 
-            return tuple(Dataset(d, self.fields)
-                         for d in (train_data, val_data, test_data) if d)
+        splits = tuple(Dataset(d, self.fields)
+                       for d in (train_data, val_data, test_data) if d)
+
+        # In case the parent sort key isn't none
+        if self.sort_key:
+            for subset in splits:
+                subset.sort_key = self.sort_key
+        return splits
 
     def __getitem__(self, i):
         return self.examples[i]

{"golden_diff": "diff --git a/torchtext/data/dataset.py b/torchtext/data/dataset.py\n--- a/torchtext/data/dataset.py\n+++ b/torchtext/data/dataset.py\n@@ -108,8 +108,6 @@\n         if not stratified:\n             train_data, test_data, val_data = rationed_split(self.examples, train_ratio,\n                                                              test_ratio, val_ratio, rnd)\n-            return tuple(Dataset(d, self.fields)\n-                         for d in (train_data, val_data, test_data) if d)\n         else:\n             if strata_field not in self.fields:\n                 raise ValueError(\"Invalid field name for strata_field {}\"\n@@ -125,8 +123,14 @@\n                 test_data += group_test\n                 val_data += group_val\n \n-            return tuple(Dataset(d, self.fields)\n-                         for d in (train_data, val_data, test_data) if d)\n+        splits = tuple(Dataset(d, self.fields)\n+                       for d in (train_data, val_data, test_data) if d)\n+\n+        # In case the parent sort key isn't none\n+        if self.sort_key:\n+            for subset in splits:\n+                subset.sort_key = self.sort_key\n+        return splits\n \n     def __getitem__(self, i):\n         return self.examples[i]\n", "issue": "dataset.sort_key not retained after dataset.split()\nHi I was trying out the new `split()` functionality to split a test set into test and validation set. When the `BucketIterator` is being set up from a newly split dataset the sorting fails because `dataset.sort_key` is None. Looking at the `split()`  function I see that a new instance of type `Dataset` is created in which `sort_key` is None by default:\r\n```\r\ndef split():\r\n...\r\nif not stratified:\r\n            train_data, test_data, val_data = rationed_split(self.examples, train_ratio,\r\n                                                             test_ratio, val_ratio, rnd)\r\n            return tuple(Dataset(d, self.fields)\r\n                         for d in (train_data, val_data, test_data) if d)\r\n```\r\n\r\nI guess one way would be to explicitly copy the `sort_key` into the new instance. A more generic way could be to make a copy of the current instance and replace only the `examples` attribute but I'm not sure if that is really needed.\r\n\r\nThanks!\n", "before_files": [{"content": "import io\nimport os\nimport zipfile\nimport tarfile\nfrom functools import partial\n\nimport torch.utils.data\n\nfrom .utils import RandomShuffler\nfrom .example import Example\nfrom ..utils import download_from_url, unicode_csv_reader\n\n\nclass Dataset(torch.utils.data.Dataset):\n    \"\"\"Defines a dataset composed of Examples along with its Fields.\n\n    Attributes:\n        sort_key (callable): A key to use for sorting dataset examples for batching\n            together examples with similar lengths to minimize padding.\n        examples (list(Example)): The examples in this dataset.\n        fields (dict[str, Field]): Contains the name of each column or field, together\n            with the corresponding Field object. Two fields with the same Field object\n            will have a shared vocabulary.\n    \"\"\"\n    sort_key = None\n\n    def __init__(self, examples, fields, filter_pred=None):\n        \"\"\"Create a dataset from a list of Examples and Fields.\n\n        Arguments:\n            examples: List of Examples.\n            fields (List(tuple(str, Field))): The Fields to use in this tuple. The\n                string is a field name, and the Field is the associated field.\n            filter_pred (callable or None): Use only examples for which\n                filter_pred(example) is True, or use all examples if None.\n                Default is None.\n        \"\"\"\n        if filter_pred is not None:\n            make_list = isinstance(examples, list)\n            examples = filter(filter_pred, examples)\n            if make_list:\n                examples = list(examples)\n        self.examples = examples\n        self.fields = dict(fields)\n        # Unpack field tuples\n        for n, f in list(self.fields.items()):\n            if isinstance(n, tuple):\n                self.fields.update(zip(n, f))\n                del self.fields[n]\n\n    @classmethod\n    def splits(cls, path=None, root='.data', train=None, validation=None,\n               test=None, **kwargs):\n        \"\"\"Create Dataset objects for multiple splits of a dataset.\n\n        Arguments:\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root (str): Root dataset storage directory. Default is '.data'.\n            train (str): Suffix to add to path for the train set, or None for no\n                train set. Default is None.\n            validation (str): Suffix to add to path for the validation set, or None\n                for no validation set. Default is None.\n            test (str): Suffix to add to path for the test set, or None for no test\n                set. Default is None.\n            Remaining keyword arguments: Passed to the constructor of the\n                Dataset (sub)class being used.\n\n        Returns:\n            Tuple[Dataset]: Datasets for train, validation, and\n                test splits in that order, if provided.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n        train_data = None if train is None else cls(\n            os.path.join(path, train), **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    def split(self, split_ratio=0.7, stratified=False, strata_field='label',\n              random_state=None):\n        \"\"\"Create train-test(-valid?) splits from the instance's examples.\n\n        Arguments:\n            split_ratio (float or List of floats): a number [0, 1] denoting the amount\n                of data to be used for the training split (rest is used for validation),\n                or a list of numbers denoting the relative sizes of train, test and valid\n                splits respectively. If the relative size for valid is missing, only the\n                train-test split is returned. Default is 0.7 (for th train set).\n            stratified (bool): whether the sampling should be stratified.\n                Default is False.\n            strata_field (str): name of the examples Field stratified over.\n                Default is 'label' for the conventional label field.\n            random_state (int): the random seed used for shuffling.\n\n        Returns:\n            Tuple[Dataset]: Datasets for train, validation, and\n                test splits in that order, if the splits are provided.\n        \"\"\"\n        train_ratio, test_ratio, val_ratio = check_split_ratio(split_ratio)\n\n        # For the permutations\n        rnd = RandomShuffler(random_state)\n        if not stratified:\n            train_data, test_data, val_data = rationed_split(self.examples, train_ratio,\n                                                             test_ratio, val_ratio, rnd)\n            return tuple(Dataset(d, self.fields)\n                         for d in (train_data, val_data, test_data) if d)\n        else:\n            if strata_field not in self.fields:\n                raise ValueError(\"Invalid field name for strata_field {}\"\n                                 .format(strata_field))\n            strata = stratify(self.examples, strata_field)\n            train_data, test_data, val_data = [], [], []\n            for group in strata:\n                # Stratify each group and add together the indices.\n                group_train, group_test, group_val = rationed_split(group, train_ratio,\n                                                                    test_ratio, val_ratio,\n                                                                    rnd)\n                train_data += group_train\n                test_data += group_test\n                val_data += group_val\n\n            return tuple(Dataset(d, self.fields)\n                         for d in (train_data, val_data, test_data) if d)\n\n    def __getitem__(self, i):\n        return self.examples[i]\n\n    def __len__(self):\n        try:\n            return len(self.examples)\n        except TypeError:\n            return 2**32\n\n    def __iter__(self):\n        for x in self.examples:\n            yield x\n\n    def __getattr__(self, attr):\n        if attr in self.fields:\n            for x in self.examples:\n                yield getattr(x, attr)\n\n    @classmethod\n    def download(cls, root, check=None):\n        \"\"\"Download and unzip an online archive (.zip, .gz, or .tgz).\n\n        Arguments:\n            root (str): Folder to download data to.\n            check (str or None): Folder whose existence indicates\n                that the dataset has already been downloaded, or\n                None to check the existence of root/{cls.name}.\n\n        Returns:\n            str: Path to extracted dataset.\n        \"\"\"\n        path = os.path.join(root, cls.name)\n        check = path if check is None else check\n        if not os.path.isdir(check):\n            for url in cls.urls:\n                if isinstance(url, tuple):\n                    url, filename = url\n                else:\n                    filename = os.path.basename(url)\n                zpath = os.path.join(path, filename)\n                if not os.path.isfile(zpath):\n                    if not os.path.exists(os.path.dirname(zpath)):\n                        os.makedirs(os.path.dirname(zpath))\n                    print('downloading {}'.format(filename))\n                    download_from_url(url, zpath)\n                ext = os.path.splitext(filename)[-1]\n                if ext == '.zip':\n                    with zipfile.ZipFile(zpath, 'r') as zfile:\n                        print('extracting')\n                        zfile.extractall(path)\n                elif ext in ['.gz', '.tgz']:\n                    with tarfile.open(zpath, 'r:gz') as tar:\n                        dirs = [member for member in tar.getmembers()]\n                        tar.extractall(path=path, members=dirs)\n        return os.path.join(path, cls.dirname)\n\n\nclass TabularDataset(Dataset):\n    \"\"\"Defines a Dataset of columns stored in CSV, TSV, or JSON format.\"\"\"\n\n    def __init__(self, path, format, fields, skip_header=False, **kwargs):\n        \"\"\"Create a TabularDataset given a path, file format, and field list.\n\n        Arguments:\n            path (str): Path to the data file.\n            format (str): The format of the data file. One of \"CSV\", \"TSV\", or\n                \"JSON\" (case-insensitive).\n            fields (list(tuple(str, Field)) or dict[str: tuple(str, Field)]:\n                If using a list, the format must be CSV or TSV, and the values of the list\n                should be tuples of (name, field).\n                The fields should be in the same order as the columns in the CSV or TSV\n                file, while tuples of (name, None) represent columns that will be ignored.\n\n                If using a dict, the keys should be a subset of the JSON keys or CSV/TSV\n                columns, and the values should be tuples of (name, field).\n                Keys not present in the input dictionary are ignored.\n                This allows the user to rename columns from their JSON/CSV/TSV key names\n                and also enables selecting a subset of columns to load.\n            skip_header (bool): Whether to skip the first line of the input file.\n        \"\"\"\n        make_example = {\n            'json': Example.fromJSON, 'dict': Example.fromdict,\n            'tsv': Example.fromCSV, 'csv': Example.fromCSV}[format.lower()]\n\n        with io.open(os.path.expanduser(path), encoding=\"utf8\") as f:\n            if format == 'csv':\n                reader = unicode_csv_reader(f)\n            elif format == 'tsv':\n                reader = unicode_csv_reader(f, delimiter='\\t')\n            else:\n                reader = f\n\n            if format in ['csv', 'tsv'] and isinstance(fields, dict):\n                if skip_header:\n                    raise ValueError('When using a dict to specify fields with a {} file,'\n                                     'skip_header must be False and'\n                                     'the file must have a header.'.format(format))\n                header = next(reader)\n                field_to_index = {f: header.index(f) for f in fields.keys()}\n                make_example = partial(make_example, field_to_index=field_to_index)\n\n            if skip_header:\n                next(reader)\n\n            examples = [make_example(line, fields) for line in reader]\n\n        if isinstance(fields, dict):\n            fields, field_dict = [], fields\n            for field in field_dict.values():\n                if isinstance(field, list):\n                    fields.extend(field)\n                else:\n                    fields.append(field)\n\n        super(TabularDataset, self).__init__(examples, fields, **kwargs)\n\n\ndef check_split_ratio(split_ratio):\n    \"\"\"Check that the split ratio argument is not malformed\"\"\"\n    valid_ratio = 0.\n    if isinstance(split_ratio, float):\n        # Only the train set relative ratio is provided\n        # Assert in bounds, validation size is zero\n        assert split_ratio > 0. and split_ratio < 1., (\n            \"Split ratio {} not between 0 and 1\".format(split_ratio))\n\n        test_ratio = 1. - split_ratio\n        return (split_ratio, test_ratio, valid_ratio)\n    elif isinstance(split_ratio, list):\n        # A list of relative ratios is provided\n        length = len(split_ratio)\n        assert length == 2 or length == 3, (\n            \"Length of split ratio list should be 2 or 3, got {}\".format(split_ratio))\n\n        # Normalize if necessary\n        ratio_sum = sum(split_ratio)\n        if not ratio_sum == 1.:\n            split_ratio = [float(ratio) / ratio_sum for ratio in split_ratio]\n\n        if length == 2:\n            return tuple(split_ratio + [valid_ratio])\n        return tuple(split_ratio)\n    else:\n        raise ValueError('Split ratio must be float or a list, got {}'\n                         .format(type(split_ratio)))\n\n\ndef stratify(examples, strata_field):\n    # The field has to be hashable otherwise this doesn't work\n    # There's two iterations over the whole dataset here, which can be\n    # reduced to just one if a dedicated method for stratified splitting is used\n    unique_strata = set(getattr(example, strata_field) for example in examples)\n    strata_maps = {s: [] for s in unique_strata}\n    for example in examples:\n        strata_maps[getattr(example, strata_field)].append(example)\n    return list(strata_maps.values())\n\n\ndef rationed_split(examples, train_ratio, test_ratio, val_ratio, rnd):\n    # Create a random permutation of examples, then split them\n    # by ratio x length slices for each of the train/test/dev? splits\n    N = len(examples)\n    randperm = rnd(range(N))\n    train_len = int(round(train_ratio * N))\n\n    # Due to possible rounding problems\n    if not val_ratio:\n        test_len = N - train_len\n    else:\n        test_len = int(round(test_ratio * N))\n\n    indices = (randperm[:train_len],  # Train\n               randperm[train_len:train_len + test_len],  # Test\n               randperm[train_len + test_len:])  # Validation\n\n    # There's a possibly empty list for the validation set\n    data = tuple([examples[i] for i in index] for index in indices)\n\n    return data\n", "path": "torchtext/data/dataset.py"}], "after_files": [{"content": "import io\nimport os\nimport zipfile\nimport tarfile\nfrom functools import partial\n\nimport torch.utils.data\n\nfrom .utils import RandomShuffler\nfrom .example import Example\nfrom ..utils import download_from_url, unicode_csv_reader\n\n\nclass Dataset(torch.utils.data.Dataset):\n    \"\"\"Defines a dataset composed of Examples along with its Fields.\n\n    Attributes:\n        sort_key (callable): A key to use for sorting dataset examples for batching\n            together examples with similar lengths to minimize padding.\n        examples (list(Example)): The examples in this dataset.\n        fields (dict[str, Field]): Contains the name of each column or field, together\n            with the corresponding Field object. Two fields with the same Field object\n            will have a shared vocabulary.\n    \"\"\"\n    sort_key = None\n\n    def __init__(self, examples, fields, filter_pred=None):\n        \"\"\"Create a dataset from a list of Examples and Fields.\n\n        Arguments:\n            examples: List of Examples.\n            fields (List(tuple(str, Field))): The Fields to use in this tuple. The\n                string is a field name, and the Field is the associated field.\n            filter_pred (callable or None): Use only examples for which\n                filter_pred(example) is True, or use all examples if None.\n                Default is None.\n        \"\"\"\n        if filter_pred is not None:\n            make_list = isinstance(examples, list)\n            examples = filter(filter_pred, examples)\n            if make_list:\n                examples = list(examples)\n        self.examples = examples\n        self.fields = dict(fields)\n        # Unpack field tuples\n        for n, f in list(self.fields.items()):\n            if isinstance(n, tuple):\n                self.fields.update(zip(n, f))\n                del self.fields[n]\n\n    @classmethod\n    def splits(cls, path=None, root='.data', train=None, validation=None,\n               test=None, **kwargs):\n        \"\"\"Create Dataset objects for multiple splits of a dataset.\n\n        Arguments:\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root (str): Root dataset storage directory. Default is '.data'.\n            train (str): Suffix to add to path for the train set, or None for no\n                train set. Default is None.\n            validation (str): Suffix to add to path for the validation set, or None\n                for no validation set. Default is None.\n            test (str): Suffix to add to path for the test set, or None for no test\n                set. Default is None.\n            Remaining keyword arguments: Passed to the constructor of the\n                Dataset (sub)class being used.\n\n        Returns:\n            Tuple[Dataset]: Datasets for train, validation, and\n                test splits in that order, if provided.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n        train_data = None if train is None else cls(\n            os.path.join(path, train), **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    def split(self, split_ratio=0.7, stratified=False, strata_field='label',\n              random_state=None):\n        \"\"\"Create train-test(-valid?) splits from the instance's examples.\n\n        Arguments:\n            split_ratio (float or List of floats): a number [0, 1] denoting the amount\n                of data to be used for the training split (rest is used for validation),\n                or a list of numbers denoting the relative sizes of train, test and valid\n                splits respectively. If the relative size for valid is missing, only the\n                train-test split is returned. Default is 0.7 (for th train set).\n            stratified (bool): whether the sampling should be stratified.\n                Default is False.\n            strata_field (str): name of the examples Field stratified over.\n                Default is 'label' for the conventional label field.\n            random_state (int): the random seed used for shuffling.\n\n        Returns:\n            Tuple[Dataset]: Datasets for train, validation, and\n                test splits in that order, if the splits are provided.\n        \"\"\"\n        train_ratio, test_ratio, val_ratio = check_split_ratio(split_ratio)\n\n        # For the permutations\n        rnd = RandomShuffler(random_state)\n        if not stratified:\n            train_data, test_data, val_data = rationed_split(self.examples, train_ratio,\n                                                             test_ratio, val_ratio, rnd)\n        else:\n            if strata_field not in self.fields:\n                raise ValueError(\"Invalid field name for strata_field {}\"\n                                 .format(strata_field))\n            strata = stratify(self.examples, strata_field)\n            train_data, test_data, val_data = [], [], []\n            for group in strata:\n                # Stratify each group and add together the indices.\n                group_train, group_test, group_val = rationed_split(group, train_ratio,\n                                                                    test_ratio, val_ratio,\n                                                                    rnd)\n                train_data += group_train\n                test_data += group_test\n                val_data += group_val\n\n        splits = tuple(Dataset(d, self.fields)\n                       for d in (train_data, val_data, test_data) if d)\n\n        # In case the parent sort key isn't none\n        if self.sort_key:\n            for subset in splits:\n                subset.sort_key = self.sort_key\n        return splits\n\n    def __getitem__(self, i):\n        return self.examples[i]\n\n    def __len__(self):\n        try:\n            return len(self.examples)\n        except TypeError:\n            return 2**32\n\n    def __iter__(self):\n        for x in self.examples:\n            yield x\n\n    def __getattr__(self, attr):\n        if attr in self.fields:\n            for x in self.examples:\n                yield getattr(x, attr)\n\n    @classmethod\n    def download(cls, root, check=None):\n        \"\"\"Download and unzip an online archive (.zip, .gz, or .tgz).\n\n        Arguments:\n            root (str): Folder to download data to.\n            check (str or None): Folder whose existence indicates\n                that the dataset has already been downloaded, or\n                None to check the existence of root/{cls.name}.\n\n        Returns:\n            str: Path to extracted dataset.\n        \"\"\"\n        path = os.path.join(root, cls.name)\n        check = path if check is None else check\n        if not os.path.isdir(check):\n            for url in cls.urls:\n                if isinstance(url, tuple):\n                    url, filename = url\n                else:\n                    filename = os.path.basename(url)\n                zpath = os.path.join(path, filename)\n                if not os.path.isfile(zpath):\n                    if not os.path.exists(os.path.dirname(zpath)):\n                        os.makedirs(os.path.dirname(zpath))\n                    print('downloading {}'.format(filename))\n                    download_from_url(url, zpath)\n                ext = os.path.splitext(filename)[-1]\n                if ext == '.zip':\n                    with zipfile.ZipFile(zpath, 'r') as zfile:\n                        print('extracting')\n                        zfile.extractall(path)\n                elif ext in ['.gz', '.tgz']:\n                    with tarfile.open(zpath, 'r:gz') as tar:\n                        dirs = [member for member in tar.getmembers()]\n                        tar.extractall(path=path, members=dirs)\n        return os.path.join(path, cls.dirname)\n\n\nclass TabularDataset(Dataset):\n    \"\"\"Defines a Dataset of columns stored in CSV, TSV, or JSON format.\"\"\"\n\n    def __init__(self, path, format, fields, skip_header=False, **kwargs):\n        \"\"\"Create a TabularDataset given a path, file format, and field list.\n\n        Arguments:\n            path (str): Path to the data file.\n            format (str): The format of the data file. One of \"CSV\", \"TSV\", or\n                \"JSON\" (case-insensitive).\n            fields (list(tuple(str, Field)) or dict[str: tuple(str, Field)]:\n                If using a list, the format must be CSV or TSV, and the values of the list\n                should be tuples of (name, field).\n                The fields should be in the same order as the columns in the CSV or TSV\n                file, while tuples of (name, None) represent columns that will be ignored.\n\n                If using a dict, the keys should be a subset of the JSON keys or CSV/TSV\n                columns, and the values should be tuples of (name, field).\n                Keys not present in the input dictionary are ignored.\n                This allows the user to rename columns from their JSON/CSV/TSV key names\n                and also enables selecting a subset of columns to load.\n            skip_header (bool): Whether to skip the first line of the input file.\n        \"\"\"\n        make_example = {\n            'json': Example.fromJSON, 'dict': Example.fromdict,\n            'tsv': Example.fromCSV, 'csv': Example.fromCSV}[format.lower()]\n\n        with io.open(os.path.expanduser(path), encoding=\"utf8\") as f:\n            if format == 'csv':\n                reader = unicode_csv_reader(f)\n            elif format == 'tsv':\n                reader = unicode_csv_reader(f, delimiter='\\t')\n            else:\n                reader = f\n\n            if format in ['csv', 'tsv'] and isinstance(fields, dict):\n                if skip_header:\n                    raise ValueError('When using a dict to specify fields with a {} file,'\n                                     'skip_header must be False and'\n                                     'the file must have a header.'.format(format))\n                header = next(reader)\n                field_to_index = {f: header.index(f) for f in fields.keys()}\n                make_example = partial(make_example, field_to_index=field_to_index)\n\n            if skip_header:\n                next(reader)\n\n            examples = [make_example(line, fields) for line in reader]\n\n        if isinstance(fields, dict):\n            fields, field_dict = [], fields\n            for field in field_dict.values():\n                if isinstance(field, list):\n                    fields.extend(field)\n                else:\n                    fields.append(field)\n\n        super(TabularDataset, self).__init__(examples, fields, **kwargs)\n\n\ndef check_split_ratio(split_ratio):\n    \"\"\"Check that the split ratio argument is not malformed\"\"\"\n    valid_ratio = 0.\n    if isinstance(split_ratio, float):\n        # Only the train set relative ratio is provided\n        # Assert in bounds, validation size is zero\n        assert split_ratio > 0. and split_ratio < 1., (\n            \"Split ratio {} not between 0 and 1\".format(split_ratio))\n\n        test_ratio = 1. - split_ratio\n        return (split_ratio, test_ratio, valid_ratio)\n    elif isinstance(split_ratio, list):\n        # A list of relative ratios is provided\n        length = len(split_ratio)\n        assert length == 2 or length == 3, (\n            \"Length of split ratio list should be 2 or 3, got {}\".format(split_ratio))\n\n        # Normalize if necessary\n        ratio_sum = sum(split_ratio)\n        if not ratio_sum == 1.:\n            split_ratio = [float(ratio) / ratio_sum for ratio in split_ratio]\n\n        if length == 2:\n            return tuple(split_ratio + [valid_ratio])\n        return tuple(split_ratio)\n    else:\n        raise ValueError('Split ratio must be float or a list, got {}'\n                         .format(type(split_ratio)))\n\n\ndef stratify(examples, strata_field):\n    # The field has to be hashable otherwise this doesn't work\n    # There's two iterations over the whole dataset here, which can be\n    # reduced to just one if a dedicated method for stratified splitting is used\n    unique_strata = set(getattr(example, strata_field) for example in examples)\n    strata_maps = {s: [] for s in unique_strata}\n    for example in examples:\n        strata_maps[getattr(example, strata_field)].append(example)\n    return list(strata_maps.values())\n\n\ndef rationed_split(examples, train_ratio, test_ratio, val_ratio, rnd):\n    # Create a random permutation of examples, then split them\n    # by ratio x length slices for each of the train/test/dev? splits\n    N = len(examples)\n    randperm = rnd(range(N))\n    train_len = int(round(train_ratio * N))\n\n    # Due to possible rounding problems\n    if not val_ratio:\n        test_len = N - train_len\n    else:\n        test_len = int(round(test_ratio * N))\n\n    indices = (randperm[:train_len],  # Train\n               randperm[train_len:train_len + test_len],  # Test\n               randperm[train_len + test_len:])  # Validation\n\n    # There's a possibly empty list for the validation set\n    data = tuple([examples[i] for i in index] for index in indices)\n\n    return data\n", "path": "torchtext/data/dataset.py"}]}