We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
ascii vs. utf-8 in torchtext/datasets/translation.py
@nelson-liu: I incorrectly brought this up in pull #52, new issue here

When trying to load splits for IWSLT (in french, german, etc...), the loading process would fail with an ascii encoding/decoding error:
```
.data/iwslt/de-en/IWSLT16.TED.dev2010.de-en.en.xml
.data/iwslt/de-en/IWSLT16.TED.tst2013.de-en.de.xml
Traceback (most recent call last):
  File "test.py", line 25, in <module>
    train, val, test = datasets.IWSLT.splits(exts=('.de', '.en'), fields=(DE, EN))
  File "build/bdist.linux-x86_64/egg/torchtext/datasets/translation.py", line 116, in splits
  File "build/bdist.linux-x86_64/egg/torchtext/datasets/translation.py", line 136, in clean
UnicodeEncodeError: 'ascii' codec can't encode character u'\xe4' in position 60: ordinal not in range(128)
```
These are my library versions:
```
numpy==1.13.3
regex==2017.9.23
spacy==1.9.0
torch==0.2.0.post4
torchtext==0.2.0b0 (just cloned a few minutes before error)
torchvision==0.1.9
```
Here is the code that I was using, from test/translation.py:
```
from torchtext import data
from torchtext import datasets

import re
import spacy
import sys

spacy_de = spacy.load('de')
spacy_en = spacy.load('en')

url = re.compile('(<url>.*</url>)')

def tokenize_de(text):
    return [tok.text for tok in spacy_de.tokenizer(url.sub('@URL@', text))]

def tokenize_en(text):
    return [tok.text for tok in spacy_en.tokenizer(url.sub('@URL@', text))]

# Testing IWSLT
DE = data.Field(tokenize=tokenize_de)
EN = data.Field(tokenize=tokenize_en)
train, val, test = datasets.IWSLT.splits(exts=('.de', '.en'), fields=(DE, EN))
```
The following fixed it for me, in torchtext/datasets/translation.py. Replace opens with io.opens specifying utf-8 for python2. It's worth noting that a friend with python3 did not have this problem.
```
127     @staticmethod
128     def clean(path):
129         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
130             print(f_xml)
131             f_txt = os.path.splitext(f_xml)[0]
132             import io
133             with io.open(f_txt, mode="w", encoding="utf-8") as fd_txt: <--- INSERT
134             #with open(f_txt, 'w') as fd_txt: <--- COMMENT
135                 root = ET.parse(f_xml).getroot()[0]
136                 for doc in root.findall('doc'):
137                     for e in doc.findall('seg'):
138                         fd_txt.write(e.text.strip() + '\n')
139         xml_tags = ['<url', '<keywords', '<talkid', '<description',
140                     '<reviewer', '<translator', '<title', '<speaker']
141         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
142             print(f_orig)
143             f_txt = f_orig.replace('.tags', '')
144             with io.open(f_txt,mode='w',encoding='utf-8') as fd_txt, io.open(f_orig,mode='r',encoding='utf=8') as fd_orig: <--- INSERT
145             #with open(f_txt, 'w') as fd_txt, open(f_orig) as fd_orig: <--- COMMENT
146                 for l in fd_orig:
147                     if not any(tag in l for tag in xml_tags):
148                         fd_txt.write(l.strip() + '\n')
```
@jekbradbury, you were correct in pull #52 that I didn't need the middle block explicitly encoding/decoding (not seen here) since the file is already open as utf-8.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchtext/datasets/translation.py`
Content:
```
1 import os
2 import xml.etree.ElementTree as ET
3 import glob
4 
5 from .. import data
6 
7 
8 class TranslationDataset(data.Dataset):
9     """Defines a dataset for machine translation."""
10 
11     @staticmethod
12     def sort_key(ex):
13         return data.interleave_keys(len(ex.src), len(ex.trg))
14 
15     def __init__(self, path, exts, fields, **kwargs):
16         """Create a TranslationDataset given paths and fields.
17 
18         Arguments:
19             path: Common prefix of paths to the data files for both languages.
20             exts: A tuple containing the extension to path for each language.
21             fields: A tuple containing the fields that will be used for data
22                 in each language.
23             Remaining keyword arguments: Passed to the constructor of
24                 data.Dataset.
25         """
26         if not isinstance(fields[0], (tuple, list)):
27             fields = [('src', fields[0]), ('trg', fields[1])]
28 
29         src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)
30 
31         examples = []
32         with open(src_path) as src_file, open(trg_path) as trg_file:
33             for src_line, trg_line in zip(src_file, trg_file):
34                 src_line, trg_line = src_line.strip(), trg_line.strip()
35                 if src_line != '' and trg_line != '':
36                     examples.append(data.Example.fromlist(
37                         [src_line, trg_line], fields))
38 
39         super(TranslationDataset, self).__init__(examples, fields, **kwargs)
40 
41 
42 class Multi30k(TranslationDataset, data.Dataset):
43     """Defines a dataset for the multi-modal WMT 2016 task"""
44 
45     urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
46             'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
47             'https://staff.fnwi.uva.nl/d.elliott/wmt16/mmt16_task1_test.tgz']
48     name = 'multi30k'
49     dirname = ''
50 
51     @classmethod
52     def splits(cls, exts, fields, root='.data',
53                train='train', val='val', test='test', **kwargs):
54         """Create dataset objects for splits of the Multi30k dataset.
55 
56         Arguments:
57 
58             root: directory containing Multi30k data
59             exts: A tuple containing the extension to path for each language.
60             fields: A tuple containing the fields that will be used for data
61                 in each language.
62             train: The prefix of the train data. Default: 'train'.
63             validation: The prefix of the validation data. Default: 'val'.
64             test: The prefix of the test data. Default: 'test'.
65             Remaining keyword arguments: Passed to the splits method of
66                 Dataset.
67         """
68         path = cls.download(root)
69 
70         train_data = None if train is None else cls(
71             os.path.join(path, train), exts, fields, **kwargs)
72         val_data = None if val is None else cls(
73             os.path.join(path, val), exts, fields, **kwargs)
74         test_data = None if test is None else cls(
75             os.path.join(path, test), exts, fields, **kwargs)
76         return tuple(d for d in (train_data, val_data, test_data)
77                      if d is not None)
78 
79 
80 class IWSLT(TranslationDataset, data.Dataset):
81     """Defines a dataset for the IWSLT 2016 task"""
82 
83     base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'
84     name = 'iwslt'
85     base_dirname = '{}-{}'
86 
87     @classmethod
88     def splits(cls, exts, fields, root='.data',
89                train='train', val='IWSLT16.TED.tst2013',
90                test='IWSLT16.TED.tst2014', **kwargs):
91         """Create dataset objects for splits of the IWSLT dataset.
92 
93         Arguments:
94 
95             root: directory containing Multi30k data
96             exts: A tuple containing the extension to path for each language.
97             fields: A tuple containing the fields that will be used for data
98                 in each language.
99             train: The prefix of the train data. Default: 'train'.
100             validation: The prefix of the validation data. Default: 'val'.
101             test: The prefix of the test data. Default: 'test'.
102             Remaining keyword arguments: Passed to the splits method of
103                 Dataset.
104         """
105         cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])
106         cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]
107         check = os.path.join(root, cls.name, cls.dirname)
108         path = cls.download(root, check=check)
109 
110         train = '.'.join([train, cls.dirname])
111         val = '.'.join([val, cls.dirname])
112         if test is not None:
113             test = '.'.join([test, cls.dirname])
114 
115         if not os.path.exists(os.path.join(path, train) + exts[0]):
116             cls.clean(path)
117 
118         train_data = None if train is None else cls(
119             os.path.join(path, train), exts, fields, **kwargs)
120         val_data = None if val is None else cls(
121             os.path.join(path, val), exts, fields, **kwargs)
122         test_data = None if test is None else cls(
123             os.path.join(path, test), exts, fields, **kwargs)
124         return tuple(d for d in (train_data, val_data, test_data)
125                      if d is not None)
126 
127     @staticmethod
128     def clean(path):
129         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
130             print(f_xml)
131             f_txt = os.path.splitext(f_xml)[0]
132             with open(f_txt, 'w') as fd_txt:
133                 root = ET.parse(f_xml).getroot()[0]
134                 for doc in root.findall('doc'):
135                     for e in doc.findall('seg'):
136                         fd_txt.write(e.text.strip() + '\n')
137 
138         xml_tags = ['<url', '<keywords', '<talkid', '<description',
139                     '<reviewer', '<translator', '<title', '<speaker']
140         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
141             print(f_orig)
142             f_txt = f_orig.replace('.tags', '')
143             with open(f_txt, 'w') as fd_txt, open(f_orig) as fd_orig:
144                 for l in fd_orig:
145                     if not any(tag in l for tag in xml_tags):
146                         fd_txt.write(l.strip() + '\n')
147
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py
--- a/torchtext/datasets/translation.py
+++ b/torchtext/datasets/translation.py
@@ -1,6 +1,7 @@
 import os
 import xml.etree.ElementTree as ET
 import glob
+import io
 
 from .. import data
 
@@ -129,7 +130,7 @@
         for f_xml in glob.iglob(os.path.join(path, '*.xml')):
             print(f_xml)
             f_txt = os.path.splitext(f_xml)[0]
-            with open(f_txt, 'w') as fd_txt:
+            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:
                 root = ET.parse(f_xml).getroot()[0]
                 for doc in root.findall('doc'):
                     for e in doc.findall('seg'):
@@ -140,7 +141,8 @@
         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):
             print(f_orig)
             f_txt = f_orig.replace('.tags', '')
-            with open(f_txt, 'w') as fd_txt, open(f_orig) as fd_orig:
+            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \
+                    io.open(f_orig, mode='w', encoding='utf-8') as fd_orig:
                 for l in fd_orig:
                     if not any(tag in l for tag in xml_tags):
                         fd_txt.write(l.strip() + '\n')

{"golden_diff": "diff --git a/torchtext/datasets/translation.py b/torchtext/datasets/translation.py\n--- a/torchtext/datasets/translation.py\n+++ b/torchtext/datasets/translation.py\n@@ -1,6 +1,7 @@\n import os\n import xml.etree.ElementTree as ET\n import glob\n+import io\n \n from .. import data\n \n@@ -129,7 +130,7 @@\n         for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n             print(f_xml)\n             f_txt = os.path.splitext(f_xml)[0]\n-            with open(f_txt, 'w') as fd_txt:\n+            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                 root = ET.parse(f_xml).getroot()[0]\n                 for doc in root.findall('doc'):\n                     for e in doc.findall('seg'):\n@@ -140,7 +141,8 @@\n         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n             print(f_orig)\n             f_txt = f_orig.replace('.tags', '')\n-            with open(f_txt, 'w') as fd_txt, open(f_orig) as fd_orig:\n+            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n+                    io.open(f_orig, mode='w', encoding='utf-8') as fd_orig:\n                 for l in fd_orig:\n                     if not any(tag in l for tag in xml_tags):\n                         fd_txt.write(l.strip() + '\\n')\n", "issue": "ascii vs. utf-8 in torchtext/datasets/translation.py\n@nelson-liu: I incorrectly brought this up in pull #52, new issue here\r\n\r\nWhen trying to load splits for IWSLT (in french, german, etc...), the loading process would fail with an ascii encoding/decoding error:\r\n```\r\n.data/iwslt/de-en/IWSLT16.TED.dev2010.de-en.en.xml\r\n.data/iwslt/de-en/IWSLT16.TED.tst2013.de-en.de.xml\r\nTraceback (most recent call last):\r\n  File \"test.py\", line 25, in <module>\r\n    train, val, test = datasets.IWSLT.splits(exts=('.de', '.en'), fields=(DE, EN))\r\n  File \"build/bdist.linux-x86_64/egg/torchtext/datasets/translation.py\", line 116, in splits\r\n  File \"build/bdist.linux-x86_64/egg/torchtext/datasets/translation.py\", line 136, in clean\r\nUnicodeEncodeError: 'ascii' codec can't encode character u'\\xe4' in position 60: ordinal not in range(128)\r\n```\r\nThese are my library versions:\r\n```\r\nnumpy==1.13.3\r\nregex==2017.9.23\r\nspacy==1.9.0\r\ntorch==0.2.0.post4\r\ntorchtext==0.2.0b0 (just cloned a few minutes before error)\r\ntorchvision==0.1.9\r\n```\r\nHere is the code that I was using, from test/translation.py:\r\n```\r\nfrom torchtext import data\r\nfrom torchtext import datasets\r\n\r\nimport re\r\nimport spacy\r\nimport sys\r\n\r\nspacy_de = spacy.load('de')\r\nspacy_en = spacy.load('en')\r\n\r\nurl = re.compile('(<url>.*</url>)')\r\n\r\ndef tokenize_de(text):\r\n    return [tok.text for tok in spacy_de.tokenizer(url.sub('@URL@', text))]\r\n\r\ndef tokenize_en(text):\r\n    return [tok.text for tok in spacy_en.tokenizer(url.sub('@URL@', text))]\r\n\r\n# Testing IWSLT\r\nDE = data.Field(tokenize=tokenize_de)\r\nEN = data.Field(tokenize=tokenize_en)\r\ntrain, val, test = datasets.IWSLT.splits(exts=('.de', '.en'), fields=(DE, EN))\r\n```\r\nThe following fixed it for me, in torchtext/datasets/translation.py. Replace opens with io.opens specifying utf-8 for python2. It's worth noting that a friend with python3 did not have this problem.\r\n```\r\n127     @staticmethod\r\n128     def clean(path):\r\n129         for f_xml in glob.iglob(os.path.join(path, '*.xml')):\r\n130             print(f_xml)\r\n131             f_txt = os.path.splitext(f_xml)[0]\r\n132             import io\r\n133             with io.open(f_txt, mode=\"w\", encoding=\"utf-8\") as fd_txt: <--- INSERT\r\n134             #with open(f_txt, 'w') as fd_txt: <--- COMMENT\r\n135                 root = ET.parse(f_xml).getroot()[0]\r\n136                 for doc in root.findall('doc'):\r\n137                     for e in doc.findall('seg'):\r\n138                         fd_txt.write(e.text.strip() + '\\n')\r\n139         xml_tags = ['<url', '<keywords', '<talkid', '<description',\r\n140                     '<reviewer', '<translator', '<title', '<speaker']\r\n141         for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\r\n142             print(f_orig)\r\n143             f_txt = f_orig.replace('.tags', '')\r\n144             with io.open(f_txt,mode='w',encoding='utf-8') as fd_txt, io.open(f_orig,mode='r',encoding='utf=8') as fd_orig: <--- INSERT\r\n145             #with open(f_txt, 'w') as fd_txt, open(f_orig) as fd_orig: <--- COMMENT\r\n146                 for l in fd_orig:\r\n147                     if not any(tag in l for tag in xml_tags):\r\n148                         fd_txt.write(l.strip() + '\\n')\r\n```\r\n@jekbradbury, you were correct in pull #52 that I didn't need the middle block explicitly encoding/decoding (not seen here) since the file is already open as utf-8.\n", "before_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n\nclass Multi30k(TranslationDataset, data.Dataset):\n    \"\"\"Defines a dataset for the multi-modal WMT 2016 task\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'https://staff.fnwi.uva.nl/d.elliott/wmt16/mmt16_task1_test.tgz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', val='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n\n            root: directory containing Multi30k data\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if val is None else cls(\n            os.path.join(path, val), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass IWSLT(TranslationDataset, data.Dataset):\n    \"\"\"Defines a dataset for the IWSLT 2016 task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', val='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n\n            root: directory containing Multi30k data\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        val = '.'.join([val, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if val is None else cls(\n            os.path.join(path, val), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with open(f_txt, 'w') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with open(f_txt, 'w') as fd_txt, open(f_orig) as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n", "path": "torchtext/datasets/translation.py"}], "after_files": [{"content": "import os\nimport xml.etree.ElementTree as ET\nimport glob\nimport io\n\nfrom .. import data\n\n\nclass TranslationDataset(data.Dataset):\n    \"\"\"Defines a dataset for machine translation.\"\"\"\n\n    @staticmethod\n    def sort_key(ex):\n        return data.interleave_keys(len(ex.src), len(ex.trg))\n\n    def __init__(self, path, exts, fields, **kwargs):\n        \"\"\"Create a TranslationDataset given paths and fields.\n\n        Arguments:\n            path: Common prefix of paths to the data files for both languages.\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            Remaining keyword arguments: Passed to the constructor of\n                data.Dataset.\n        \"\"\"\n        if not isinstance(fields[0], (tuple, list)):\n            fields = [('src', fields[0]), ('trg', fields[1])]\n\n        src_path, trg_path = tuple(os.path.expanduser(path + x) for x in exts)\n\n        examples = []\n        with open(src_path) as src_file, open(trg_path) as trg_file:\n            for src_line, trg_line in zip(src_file, trg_file):\n                src_line, trg_line = src_line.strip(), trg_line.strip()\n                if src_line != '' and trg_line != '':\n                    examples.append(data.Example.fromlist(\n                        [src_line, trg_line], fields))\n\n        super(TranslationDataset, self).__init__(examples, fields, **kwargs)\n\n\nclass Multi30k(TranslationDataset, data.Dataset):\n    \"\"\"Defines a dataset for the multi-modal WMT 2016 task\"\"\"\n\n    urls = ['http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n            'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n            'https://staff.fnwi.uva.nl/d.elliott/wmt16/mmt16_task1_test.tgz']\n    name = 'multi30k'\n    dirname = ''\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', val='val', test='test', **kwargs):\n        \"\"\"Create dataset objects for splits of the Multi30k dataset.\n\n        Arguments:\n\n            root: directory containing Multi30k data\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        path = cls.download(root)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if val is None else cls(\n            os.path.join(path, val), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n\nclass IWSLT(TranslationDataset, data.Dataset):\n    \"\"\"Defines a dataset for the IWSLT 2016 task\"\"\"\n\n    base_url = 'https://wit3.fbk.eu/archive/2016-01//texts/{}/{}/{}.tgz'\n    name = 'iwslt'\n    base_dirname = '{}-{}'\n\n    @classmethod\n    def splits(cls, exts, fields, root='.data',\n               train='train', val='IWSLT16.TED.tst2013',\n               test='IWSLT16.TED.tst2014', **kwargs):\n        \"\"\"Create dataset objects for splits of the IWSLT dataset.\n\n        Arguments:\n\n            root: directory containing Multi30k data\n            exts: A tuple containing the extension to path for each language.\n            fields: A tuple containing the fields that will be used for data\n                in each language.\n            train: The prefix of the train data. Default: 'train'.\n            validation: The prefix of the validation data. Default: 'val'.\n            test: The prefix of the test data. Default: 'test'.\n            Remaining keyword arguments: Passed to the splits method of\n                Dataset.\n        \"\"\"\n        cls.dirname = cls.base_dirname.format(exts[0][1:], exts[1][1:])\n        cls.urls = [cls.base_url.format(exts[0][1:], exts[1][1:], cls.dirname)]\n        check = os.path.join(root, cls.name, cls.dirname)\n        path = cls.download(root, check=check)\n\n        train = '.'.join([train, cls.dirname])\n        val = '.'.join([val, cls.dirname])\n        if test is not None:\n            test = '.'.join([test, cls.dirname])\n\n        if not os.path.exists(os.path.join(path, train) + exts[0]):\n            cls.clean(path)\n\n        train_data = None if train is None else cls(\n            os.path.join(path, train), exts, fields, **kwargs)\n        val_data = None if val is None else cls(\n            os.path.join(path, val), exts, fields, **kwargs)\n        test_data = None if test is None else cls(\n            os.path.join(path, test), exts, fields, **kwargs)\n        return tuple(d for d in (train_data, val_data, test_data)\n                     if d is not None)\n\n    @staticmethod\n    def clean(path):\n        for f_xml in glob.iglob(os.path.join(path, '*.xml')):\n            print(f_xml)\n            f_txt = os.path.splitext(f_xml)[0]\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt:\n                root = ET.parse(f_xml).getroot()[0]\n                for doc in root.findall('doc'):\n                    for e in doc.findall('seg'):\n                        fd_txt.write(e.text.strip() + '\\n')\n\n        xml_tags = ['<url', '<keywords', '<talkid', '<description',\n                    '<reviewer', '<translator', '<title', '<speaker']\n        for f_orig in glob.iglob(os.path.join(path, 'train.tags*')):\n            print(f_orig)\n            f_txt = f_orig.replace('.tags', '')\n            with io.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \\\n                    io.open(f_orig, mode='w', encoding='utf-8') as fd_orig:\n                for l in fd_orig:\n                    if not any(tag in l for tag in xml_tags):\n                        fd_txt.write(l.strip() + '\\n')\n", "path": "torchtext/datasets/translation.py"}]}