We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Unicode Error in Using IWSLT dataset: TypeError: write() argument 1 must be unicode, not str
Hi,

I'm using the IWSLT translation dataset in torchtext. However I found the following encoding errors. The code snippet is:

MAX_LEN = 100
train, val, test = datasets.IWSLT.splits(
        exts=('.de', '.en'), fields=(SRC, TGT),
        filter_pred=lambda x: len(vars(x)['src']) <= MAX_LEN and
            len(vars(x)['trg']) <= MAX_LEN)

Indeed in an third-party implementation of Transformer using pytorch: https://github.com/harvardnlp/annotated-transformer/blob/master/The%20Annotated%20Transformer.ipynb

The error is:

.data/iwslt/de-en/IWSLT16.TED.tst2011.de-en.en.xml
Traceback (most recent call last):
  File "The+Annotated+Transformer.py", line 773, in <module>
    filter_pred=lambda x: len(vars(x)['src']) <= MAX_LEN and
  File "/usr/anaconda2/lib/python2.7/site-packages/torchtext/datasets/translation.py", line 140, in splits
    cls.clean(path)
  File "/usr/anaconda2/lib/python2.7/site-packages/torchtext/datasets/translation.py", line 160, in clean
    fd_txt.write(e.text.strip() + '\n')
TypeError: write() argument 1 must be unicode, not str

Would you help to check the issue? Thanks!

Systems config: Ubuntu 14.04, python 2.7.

Best,
Fei

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/datasets/translation.py`
Content:
```
1 import os
2 import xml.etree.ElementTree as ET
3 import glob
4 import io
5 
6 from .. import data
7 
8 
9 class TranslationDataset(data.Dataset):
10     """Defines a dataset for machine translation."""
11 
12     @staticmethod
13     def sort_key(ex):
14         return data.interleave_keys(len(ex.src), len(ex.trg))
15 
16     def __init__(self, path, exts, fields, **kwargs):
17         """Create a TranslationDataset given paths and fields.
18 
19         Arguments:
20             path: Common prefix of paths to the data files for both languages.
21             exts: A tuple containing the extension to path for each language.
22             fields: A tuple containing the fields that will be used for data
23                 in each language.
24             Remaining keyword arguments: Passed to the constructor of
25                 data.Dataset.
26         """
27         if not isinstance(fields[0], (tuple, list)):
28             fields = [('src', fields[0]), ('trg', fields[1])]
29 
30         src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)
31 
32         examples = []
33         with open(src_path) as src_file, open(trg_path) as trg_file:
34             for src_line, trg_line in zip(src_file, trg_file):
35                 src_line, trg_line = src_line.strip(), trg_line.strip()
36                 if src_line != '' and trg_line != '':
37                     examples.append(data.Example.fromlist(
38                         [src_line, trg_line], fields))
39 
40         super(TranslationDataset, self).__init__(examples, fields, **kwargs)
41 
42     @classmethod
43     def splits(cls, exts, fields, path=None, root='.data',
44                train='train', validation='val', test='test', **kwargs):
45         """Create dataset objects for splits of a TranslationDataset.
46 
47         Arguments:
48             exts: A tuple containing the extension to path for each language.
49             fields: A tuple containing the fields that will be used for data
50                 in each language.
51             path (str): Common prefix of the splits' file paths, or None to use
52                 the result of cls.download(root).
53             root: Root dataset storage directory. Default is '.data'.
54             train: The prefix of the train data. Default: 'train'.
55             validation: The prefix of the validation data. Default: 'val'.
56             test: The prefix of the test data. Default: 'test'.
57             Remaining keyword arguments: Passed to the splits method of
58                 Dataset.
59         """
60         if path is None:
61             path = cls.download(root)
62 
63         train_data = None if train is None else cls(
64             os.path.join(path, train), exts, fields, **kwargs)
65         val_data = None if validation is None else cls(
66             os.path.join(path, validation), exts, fields, **kwargs)
67         test_data = None if test is None else cls(
68             os.path.join(path, test), exts, fields, **kwargs)
69         return tuple(d for d in (train_data, val_data, test_data)
70                      if d is not None)
71 
72 
73 class Multi30k(TranslationDataset):
74     """The small-dataset WMT 2016 multimodal task, also known as Flickr30k"""
75 
76     urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
77             'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
78             'http://www.quest.dcs.shef.ac.uk/'
79             'wmt17_files_mmt/mmt_task1_test2016.tar.gz']
80     name = 'multi30k'
81     dirname = ''
82 
83     @classmethod
84     def splits(cls, exts, fields, root='.data',
85                train='train', validation='val', test='test2016', **kwargs):
86         """Create dataset objects for splits of the Multi30k dataset.
87 
88         Arguments:
89             exts: A tuple containing the extension to path for each language.
90             fields: A tuple containing the fields that will be used for data
91                 in each language.
92             root: Root dataset storage directory. Default is '.data'.
93             train: The prefix of the train data. Default: 'train'.
94             validation: The prefix of the validation data. Default: 'val'.
95             test: The prefix of the test data. Default: 'test'.
96             Remaining keyword arguments: Passed to the splits method of
97                 Dataset.
98         """
99 
100         # TODO: This is a _HORRIBLE_ patch related to #208
101         # 'path' can be passed as a kwarg to the translation dataset constructor
102         # or has to be set (so the download wouldn't be duplicated). A good idea
103         # seems to rename the existence check variable from path to something else
104         if 'path' not in kwargs:
105             expected_folder = os.path.join(root, cls.name)
106             path = expected_folder if os.path.exists(expected_folder) else None
107         else:
108             path = kwargs['path']
109             del kwargs['path']
110 
111         return super(Multi30k, cls).splits(
112             exts, fields, path, root, train, validation, test, **kwargs)
113 
114 
115 class IWSLT(TranslationDataset):
116     """The IWSLT 2016 TED talk translation task"""
117 
118     base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'
119     name = 'iwslt'
120     base_dirname = '{}-{}'
121 
122     @classmethod
123     def splits(cls, exts, fields, root='.data',
124                train='train', validation='IWSLT16.TED.tst2013',
125                test='IWSLT16.TED.tst2014', **kwargs):
126         """Create dataset objects for splits of the IWSLT dataset.
127 
128         Arguments:
129             exts: A tuple containing the extension to path for each language.
130             fields: A tuple containing the fields that will be used for data
131                 in each language.
132             root: Root dataset storage directory. Default is '.data'.
133             train: The prefix of the train data. Default: 'train'.
134             validation: The prefix of the validation data. Default: 'val'.
135             test: The prefix of the test data. Default: 'test'.
136             Remaining keyword arguments: Passed to the splits method of
137                 Dataset.
138         """
139         cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])
140         cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]
141         check = os.path.join(root, cls.name, cls.dirname)
142         path = cls.download(root, check=check)
143 
144         train = '.'.join([train, cls.dirname])
145         validation = '.'.join([validation, cls.dirname])
146         if test is not None:
147             test = '.'.join([test, cls.dirname])
148 
149         if not os.path.exists(os.path.join(path, train) + exts[0]):
150             cls.clean(path)
151 
152         train_data = None if train is None else cls(
153             os.path.join(path, train), exts, fields, **kwargs)
154         val_data = None if validation is None else cls(
155             os.path.join(path, validation), exts, fields, **kwargs)
156         test_data = None if test is None else cls(
157             os.path.join(path, test), exts, fields, **kwargs)
158         return tuple(d for d in (train_data, val_data, test_data)
159                      if d is not None)
160 
161     @staticmethod
162     def clean(path):
163         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
164             print(f_xml)
165             f_txt = os.path.splitext(f_xml)[0]
166             with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
167                 root = ET.parse(f_xml).getroot()[0]
168                 for doc in root.findall('doc'):
169                     for e in doc.findall('seg'):
170                         fd_txt.write(e.text.strip() + '\n')
171 
172         xml_tags = ['<url', '<keywords', '<talkid', '<description',
173                     '<reviewer', '<translator', '<title', '<speaker']
174         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
175             print(f_orig)
176             f_txt = f_orig.replace('.tags', '')
177             with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
178                     io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:
179                 for l in fd_orig:
180                     if not any(tag in l for tag in xml_tags):
181                         fd_txt.write(l.strip() + '\n')
182 
183 
184 class WMT14(TranslationDataset):
185     """The WMT 2014 English-German dataset, as preprocessed by Google Brain.
186 
187     Though this download contains test sets from 2015 and 2016, the train set
188     differs slightly from WMT 2015 and 2016 and significantly from WMT 2017."""
189 
190     urls = [('https://drive.google.com/uc?export=download&'
191              'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]
192     name = 'wmt14'
193     dirname = ''
194 
195     @classmethod
196     def splits(cls, exts, fields, root='.data',
197                train='train.tok.clean.bpe.32000',
198                validation='newstest2013.tok.bpe.32000',
199                test='newstest2014.tok.bpe.32000', **kwargs):
200         """Create dataset objects for splits of the WMT 2014 dataset.
201 
202         Arguments:
203             exts: A tuple containing the extensions for each language. Must be
204                 either ('.en', '.de') or the reverse.
205             fields: A tuple containing the fields that will be used for data
206                 in each language.
207             root: Root dataset storage directory. Default is '.data'.
208             train: The prefix of the train data. Default:
209                 'train.tok.clean.bpe.32000'.
210             validation: The prefix of the validation data. Default:
211                 'newstest2013.tok.bpe.32000'.
212             test: The prefix of the test data. Default:
213                 'newstest2014.tok.bpe.32000'.
214             Remaining keyword arguments: Passed to the splits method of
215                 Dataset.
216         """
217         # TODO: This is a _HORRIBLE_ patch related to #208
218         # 'path' can be passed as a kwarg to the translation dataset constructor
219         # or has to be set (so the download wouldn't be duplicated). A good idea
220         # seems to rename the existence check variable from path to something else
221         if 'path' not in kwargs:
222             expected_folder = os.path.join(root, cls.name)
223             path = expected_folder if os.path.exists(expected_folder) else None
224         else:
225             path = kwargs['path']
226             del kwargs['path']
227 
228         return super(WMT14, cls).splits(
229             exts, fields, path, root, train, validation, test, **kwargs)
230
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py
--- a/torchtext/datasets/translation.py
+++ b/torchtext/datasets/translation.py
@@ -2,6 +2,7 @@
 import xml.etree.ElementTree as ET
 import glob
 import io
+import codecs
 
 from .. import data
 
@@ -163,7 +164,7 @@
         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
             print(f_xml)
             f_txt = os.path.splitext(f_xml)[0]
-            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
+            with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
                 root = ET.parse(f_xml).getroot()[0]
                 for doc in root.findall('doc'):
                     for e in doc.findall('seg'):
@@ -174,7 +175,7 @@
         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
             print(f_orig)
             f_txt = f_orig.replace('.tags', '')
-            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
+            with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
                     io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:
                 for l in fd_orig:
                     if not any(tag in l for tag in xml_tags):

{"golden_diff": "diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py\n--- a/torchtext/datasets/translation.py\n+++ b/torchtext/datasets/translation.py\n@@ -2,6 +2,7 @@\n import xml.etree.ElementTree as ET\n import glob\n import io\n+import codecs\n \n from .. import data\n \n@@ -163,7 +164,7 @@\n         for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n             print(f_xml)\n             f_txt = os.path.splitext(f_xml)[0]\n-            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n+            with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                 root = ET.parse(f_xml).getroot()[0]\n                 for doc in root.findall('doc'):\n                     for e in doc.findall('seg'):\n@@ -174,7 +175,7 @@\n         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n             print(f_orig)\n             f_txt = f_orig.replace('.tags', '')\n-            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n+            with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                     io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                 for l in fd_orig:\n                     if not any(tag in l for tag in xml_tags):\n", "issue": "Unicode Error in Using IWSLT dataset: TypeError: write() argument 1 must be unicode, not str\nHi,\r\n\r\nI'm using the IWSLT translation dataset in torchtext. However I found the following encoding errors. The code snippet is:\r\n\r\nMAX_LEN = 100\r\ntrain, val, test = datasets.IWSLT.splits(\r\n        exts=('.de', '.en'), fields=(SRC, TGT),\r\n        filter_pred=lambda x: len(vars(x)['src']) <= MAX_LEN and\r\n            len(vars(x)['trg']) <= MAX_LEN)\r\n\r\nIndeed in an third-party implementation of Transformer using pytorch: https://github.com/harvardnlp/annotated-transformer/blob/master/The%20Annotated%20Transformer.ipynb\r\n\r\nThe error is:\r\n\r\n.data/iwslt/de-en/IWSLT16.TED.tst2011.de-en.en.xml\r\nTraceback (most recent call last):\r\n  File \"The+Annotated+Transformer.py\", line 773, in <module>\r\n    filter_pred=lambda x: len(vars(x)['src']) <= MAX_LEN and\r\n  File \"/usr/anaconda2/lib/python2.7/site-packages/torchtext/datasets/translation.py\", line 140, in splits\r\n    cls.clean(path)\r\n  File \"/usr/anaconda2/lib/python2.7/site-packages/torchtext/datasets/translation.py\", line 160, in clean\r\n    fd_txt.write(e.text.strip() + '\\n')\r\nTypeError: write() argument 1 must be unicode, not str\r\n\r\nWould you help to check the issue? Thanks!\r\n\r\nSystems config: Ubuntu 14.04, python 2.7.\r\n\r\nBest,\r\nFei\n", "before_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n    @classmethod\n    def splits(cls, exts, fields, path=None, root='.data',\n               train='train', validation='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of a TranslationDataset.\n\n        Arguments:\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass Multi30k(TranslationDataset):\n    \"\"\"The small-dataset WMT 2016 multimodal task, also known as Flickr30k\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/'\n            'wmt17_files_mmt/mmt_task1_test2016.tar.gz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='val', test='test2016', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n\n        # TODO: This is a _HORRIBLE_ patch related to #208\n        # 'path' can be passed as a kwarg to the translation dataset constructor\n        # or has to be set (so the download wouldn't be duplicated). A good idea\n        # seems to rename the existence check variable from path to something else\n        if 'path' not in kwargs:\n            expected_folder = os.path.join(root, cls.name)\n            path = expected_folder if os.path.exists(expected_folder) else None\n        else:\n            path = kwargs['path']\n            del kwargs['path']\n\n        return super(Multi30k, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n\n\nclass IWSLT(TranslationDataset):\n    \"\"\"The IWSLT 2016 TED talk translation task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        validation = '.'.join([validation, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n\n\nclass WMT14(TranslationDataset):\n    \"\"\"The WMT 2014 English-German dataset, as preprocessed by Google Brain.\n\n    Though this download contains test sets from 2015 and 2016, the train set\n    differs slightly from WMT 2015 and 2016 and significantly from WMT 2017.\"\"\"\n\n    urls = [('https://drive.google.com/uc?export=download&'\n             'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]\n    name = 'wmt14'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train.tok.clean.bpe.32000',\n               validation='newstest2013.tok.bpe.32000',\n               test='newstest2014.tok.bpe.32000', **kwargs):\n        \"\"\"Create dataset objects for splits of the WMT 2014 dataset.\n\n        Arguments:\n            exts: A tuple containing the extensions for each language. Must be\n                either ('.en', '.de') or the reverse.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default:\n                'train.tok.clean.bpe.32000'.\n            validation: The prefix of the validation data. Default:\n                'newstest2013.tok.bpe.32000'.\n            test: The prefix of the test data. Default:\n                'newstest2014.tok.bpe.32000'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        # TODO: This is a _HORRIBLE_ patch related to #208\n        # 'path' can be passed as a kwarg to the translation dataset constructor\n        # or has to be set (so the download wouldn't be duplicated). A good idea\n        # seems to rename the existence check variable from path to something else\n        if 'path' not in kwargs:\n            expected_folder = os.path.join(root, cls.name)\n            path = expected_folder if os.path.exists(expected_folder) else None\n        else:\n            path = kwargs['path']\n            del kwargs['path']\n\n        return super(WMT14, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n", "path": "torchtext/datasets/translation.py"}], "after_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\nimport codecs\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n    @classmethod\n    def splits(cls, exts, fields, path=None, root='.data',\n               train='train', validation='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of a TranslationDataset.\n\n        Arguments:\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            path (str): Common prefix of the splits' file paths, or None to use\n                the result of cls.download(root).\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        if path is None:\n            path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass Multi30k(TranslationDataset):\n    \"\"\"The small-dataset WMT 2016 multimodal task, also known as Flickr30k\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/'\n            'wmt17_files_mmt/mmt_task1_test2016.tar.gz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='val', test='test2016', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n\n        # TODO: This is a _HORRIBLE_ patch related to #208\n        # 'path' can be passed as a kwarg to the translation dataset constructor\n        # or has to be set (so the download wouldn't be duplicated). A good idea\n        # seems to rename the existence check variable from path to something else\n        if 'path' not in kwargs:\n            expected_folder = os.path.join(root, cls.name)\n            path = expected_folder if os.path.exists(expected_folder) else None\n        else:\n            path = kwargs['path']\n            del kwargs['path']\n\n        return super(Multi30k, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n\n\nclass IWSLT(TranslationDataset):\n    \"\"\"The IWSLT 2016 TED talk translation task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', validation='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        validation = '.'.join([validation, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if validation is None else cls(\n            os.path.join(path, validation), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='r', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n\n\nclass WMT14(TranslationDataset):\n    \"\"\"The WMT 2014 English-German dataset, as preprocessed by Google Brain.\n\n    Though this download contains test sets from 2015 and 2016, the train set\n    differs slightly from WMT 2015 and 2016 and significantly from WMT 2017.\"\"\"\n\n    urls = [('https://drive.google.com/uc?export=download&'\n             'id=0B_bZck-ksdkpM25jRUN2X2UxMm8', 'wmt16_en_de.tar.gz')]\n    name = 'wmt14'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train.tok.clean.bpe.32000',\n               validation='newstest2013.tok.bpe.32000',\n               test='newstest2014.tok.bpe.32000', **kwargs):\n        \"\"\"Create dataset objects for splits of the WMT 2014 dataset.\n\n        Arguments:\n            exts: A tuple containing the extensions for each language. Must be\n                either ('.en', '.de') or the reverse.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            root: Root dataset storage directory. Default is '.data'.\n            train: The prefix of the train data. Default:\n                'train.tok.clean.bpe.32000'.\n            validation: The prefix of the validation data. Default:\n                'newstest2013.tok.bpe.32000'.\n            test: The prefix of the test data. Default:\n                'newstest2014.tok.bpe.32000'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        # TODO: This is a _HORRIBLE_ patch related to #208\n        # 'path' can be passed as a kwarg to the translation dataset constructor\n        # or has to be set (so the download wouldn't be duplicated). A good idea\n        # seems to rename the existence check variable from path to something else\n        if 'path' not in kwargs:\n            expected_folder = os.path.join(root, cls.name)\n            path = expected_folder if os.path.exists(expected_folder) else None\n        else:\n            path = kwargs['path']\n            del kwargs['path']\n\n        return super(WMT14, cls).splits(\n            exts, fields, path, root, train, validation, test, **kwargs)\n", "path": "torchtext/datasets/translation.py"}]}