We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
doc for bg_iterator et al
The [documentation](https://pytorch.org/audio/datasets.html) for torchaudio does not include `bg_iterator` and `disk_cache_iterator` from [code](https://github.com/pytorch/audio/blob/master/torchaudio/datasets/utils.py). Thanks to [comment](https://discuss.pytorch.org/t/audio-dataset-load-large-file-into-memory-in-background/85943) to point this out :)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchaudio/datasets/__init__.py`
Content:
```
1 from .commonvoice import COMMONVOICE
2 from .librispeech import LIBRISPEECH
3 from .speechcommands import SPEECHCOMMANDS
4 from .utils import bg_iterator, diskcache_iterator
5 from .vctk import VCTK_092
6 from .gtzan import GTZAN
7 from .yesno import YESNO
8 from .ljspeech import LJSPEECH
9 from .cmuarctic import CMUARCTIC
10 from .cmudict import CMUDict
11 from .libritts import LIBRITTS
12 from .tedlium import TEDLIUM
13 
14 
15 __all__ = [
16     "COMMONVOICE",
17     "LIBRISPEECH",
18     "SPEECHCOMMANDS",
19     "VCTK_092",
20     "YESNO",
21     "LJSPEECH",
22     "GTZAN",
23     "CMUARCTIC",
24     "CMUDict",
25     "LIBRITTS",
26     "diskcache_iterator",
27     "bg_iterator",
28     "TEDLIUM",
29 ]
30
```

Path: `torchaudio/datasets/utils.py`
Content:
```
1 import hashlib
2 import logging
3 import os
4 import tarfile
5 import threading
6 import urllib
7 import urllib.request
8 import zipfile
9 from queue import Queue
10 from typing import Any, Iterable, List, Optional
11 
12 import torch
13 from torch.utils.data import Dataset
14 from torch.utils.model_zoo import tqdm
15 
16 from torchaudio._internal.module_utils import deprecated
17 
18 
19 def stream_url(url: str,
20                start_byte: Optional[int] = None,
21                block_size: int = 32 * 1024,
22                progress_bar: bool = True) -> Iterable:
23     """Stream url by chunk
24 
25     Args:
26         url (str): Url.
27         start_byte (int or None, optional): Start streaming at that point (Default: ``None``).
28         block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``).
29         progress_bar (bool, optional): Display a progress bar (Default: ``True``).
30     """
31 
32     # If we already have the whole file, there is no need to download it again
33     req = urllib.request.Request(url, method="HEAD")
34     with urllib.request.urlopen(req) as response:
35         url_size = int(response.info().get("Content-Length", -1))
36     if url_size == start_byte:
37         return
38 
39     req = urllib.request.Request(url)
40     if start_byte:
41         req.headers["Range"] = "bytes={}-".format(start_byte)
42 
43     with urllib.request.urlopen(req) as upointer, tqdm(
44             unit="B",
45             unit_scale=True,
46             unit_divisor=1024,
47             total=url_size,
48             disable=not progress_bar,
49     ) as pbar:
50 
51         num_bytes = 0
52         while True:
53             chunk = upointer.read(block_size)
54             if not chunk:
55                 break
56             yield chunk
57             num_bytes += len(chunk)
58             pbar.update(len(chunk))
59 
60 
61 def download_url(url: str,
62                  download_folder: str,
63                  filename: Optional[str] = None,
64                  hash_value: Optional[str] = None,
65                  hash_type: str = "sha256",
66                  progress_bar: bool = True,
67                  resume: bool = False) -> None:
68     """Download file to disk.
69 
70     Args:
71         url (str): Url.
72         download_folder (str): Folder to download file.
73         filename (str or None, optional): Name of downloaded file. If None, it is inferred from the url
74             (Default: ``None``).
75         hash_value (str or None, optional): Hash for url (Default: ``None``).
76         hash_type (str, optional): Hash type, among "sha256" and "md5" (Default: ``"sha256"``).
77         progress_bar (bool, optional): Display a progress bar (Default: ``True``).
78         resume (bool, optional): Enable resuming download (Default: ``False``).
79     """
80 
81     req = urllib.request.Request(url, method="HEAD")
82     req_info = urllib.request.urlopen(req).info()
83 
84     # Detect filename
85     filename = filename or req_info.get_filename() or os.path.basename(url)
86     filepath = os.path.join(download_folder, filename)
87     if resume and os.path.exists(filepath):
88         mode = "ab"
89         local_size: Optional[int] = os.path.getsize(filepath)
90 
91     elif not resume and os.path.exists(filepath):
92         raise RuntimeError(
93             "{} already exists. Delete the file manually and retry.".format(filepath)
94         )
95     else:
96         mode = "wb"
97         local_size = None
98 
99     if hash_value and local_size == int(req_info.get("Content-Length", -1)):
100         with open(filepath, "rb") as file_obj:
101             if validate_file(file_obj, hash_value, hash_type):
102                 return
103         raise RuntimeError(
104             "The hash of {} does not match. Delete the file manually and retry.".format(
105                 filepath
106             )
107         )
108 
109     with open(filepath, mode) as fpointer:
110         for chunk in stream_url(url, start_byte=local_size, progress_bar=progress_bar):
111             fpointer.write(chunk)
112 
113     with open(filepath, "rb") as file_obj:
114         if hash_value and not validate_file(file_obj, hash_value, hash_type):
115             raise RuntimeError(
116                 "The hash of {} does not match. Delete the file manually and retry.".format(
117                     filepath
118                 )
119             )
120 
121 
122 def validate_file(file_obj: Any, hash_value: str, hash_type: str = "sha256") -> bool:
123     """Validate a given file object with its hash.
124 
125     Args:
126         file_obj: File object to read from.
127         hash_value (str): Hash for url.
128         hash_type (str, optional): Hash type, among "sha256" and "md5" (Default: ``"sha256"``).
129 
130     Returns:
131         bool: return True if its a valid file, else False.
132     """
133 
134     if hash_type == "sha256":
135         hash_func = hashlib.sha256()
136     elif hash_type == "md5":
137         hash_func = hashlib.md5()
138     else:
139         raise ValueError
140 
141     while True:
142         # Read by chunk to avoid filling memory
143         chunk = file_obj.read(1024 ** 2)
144         if not chunk:
145             break
146         hash_func.update(chunk)
147 
148     return hash_func.hexdigest() == hash_value
149 
150 
151 def extract_archive(from_path: str, to_path: Optional[str] = None, overwrite: bool = False) -> List[str]:
152     """Extract archive.
153     Args:
154         from_path (str): the path of the archive.
155         to_path (str or None, optional): the root path of the extraced files (directory of from_path)
156             (Default: ``None``)
157         overwrite (bool, optional): overwrite existing files (Default: ``False``)
158 
159     Returns:
160         list: List of paths to extracted files even if not overwritten.
161 
162     Examples:
163         >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'
164         >>> from_path = './validation.tar.gz'
165         >>> to_path = './'
166         >>> torchaudio.datasets.utils.download_from_url(url, from_path)
167         >>> torchaudio.datasets.utils.extract_archive(from_path, to_path)
168     """
169 
170     if to_path is None:
171         to_path = os.path.dirname(from_path)
172 
173     try:
174         with tarfile.open(from_path, "r") as tar:
175             logging.info("Opened tar file {}.".format(from_path))
176             files = []
177             for file_ in tar:  # type: Any
178                 file_path = os.path.join(to_path, file_.name)
179                 if file_.isfile():
180                     files.append(file_path)
181                     if os.path.exists(file_path):
182                         logging.info("{} already extracted.".format(file_path))
183                         if not overwrite:
184                             continue
185                 tar.extract(file_, to_path)
186             return files
187     except tarfile.ReadError:
188         pass
189 
190     try:
191         with zipfile.ZipFile(from_path, "r") as zfile:
192             logging.info("Opened zip file {}.".format(from_path))
193             files = zfile.namelist()
194             for file_ in files:
195                 file_path = os.path.join(to_path, file_)
196                 if os.path.exists(file_path):
197                     logging.info("{} already extracted.".format(file_path))
198                     if not overwrite:
199                         continue
200                 zfile.extract(file_, to_path)
201         return files
202     except zipfile.BadZipFile:
203         pass
204 
205     raise NotImplementedError("We currently only support tar.gz, tgz, and zip achives.")
206 
207 
208 class _DiskCache(Dataset):
209     """
210     Wrap a dataset so that, whenever a new item is returned, it is saved to disk.
211     """
212 
213     def __init__(self, dataset: Dataset, location: str = ".cached") -> None:
214         self.dataset = dataset
215         self.location = location
216 
217         self._id = id(self)
218         self._cache: List = [None] * len(dataset)
219 
220     def __getitem__(self, n: int) -> Any:
221         if self._cache[n]:
222             f = self._cache[n]
223             return torch.load(f)
224 
225         f = str(self._id) + "-" + str(n)
226         f = os.path.join(self.location, f)
227         item = self.dataset[n]
228 
229         self._cache[n] = f
230         os.makedirs(self.location, exist_ok=True)
231         torch.save(item, f)
232 
233         return item
234 
235     def __len__(self) -> int:
236         return len(self.dataset)
237 
238 
239 @deprecated('', version='0.11')
240 def diskcache_iterator(dataset: Dataset, location: str = ".cached") -> Dataset:
241     return _DiskCache(dataset, location)
242 
243 
244 class _ThreadedIterator(threading.Thread):
245     """
246     Prefetch the next queue_length items from iterator in a background thread.
247 
248     Example:
249     >> for i in bg_iterator(range(10)):
250     >>     print(i)
251     """
252 
253     class _End:
254         pass
255 
256     def __init__(self, generator: Iterable, maxsize: int) -> None:
257         threading.Thread.__init__(self)
258         self.queue: Queue = Queue(maxsize)
259         self.generator = generator
260         self.daemon = True
261         self.start()
262 
263     def run(self) -> None:
264         for item in self.generator:
265             self.queue.put(item)
266         self.queue.put(self._End)
267 
268     def __iter__(self) -> Any:
269         return self
270 
271     def __next__(self) -> Any:
272         next_item = self.queue.get()
273         if next_item == self._End:
274             raise StopIteration
275         return next_item
276 
277     # Required for Python 2.7 compatibility
278     def next(self) -> Any:
279         return self.__next__()
280 
281 
282 @deprecated('', version='0.11')
283 def bg_iterator(iterable: Iterable, maxsize: int) -> Any:
284     return _ThreadedIterator(iterable, maxsize=maxsize)
285
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchaudio/datasets/__init__.py b/torchaudio/datasets/__init__.py
--- a/torchaudio/datasets/__init__.py
+++ b/torchaudio/datasets/__init__.py
@@ -1,7 +1,6 @@
 from .commonvoice import COMMONVOICE
 from .librispeech import LIBRISPEECH
 from .speechcommands import SPEECHCOMMANDS
-from .utils import bg_iterator, diskcache_iterator
 from .vctk import VCTK_092
 from .gtzan import GTZAN
 from .yesno import YESNO
@@ -23,7 +22,5 @@
     "CMUARCTIC",
     "CMUDict",
     "LIBRITTS",
-    "diskcache_iterator",
-    "bg_iterator",
     "TEDLIUM",
 ]
diff --git a/torchaudio/datasets/utils.py b/torchaudio/datasets/utils.py
--- a/torchaudio/datasets/utils.py
+++ b/torchaudio/datasets/utils.py
@@ -2,19 +2,13 @@
 import logging
 import os
 import tarfile
-import threading
 import urllib
 import urllib.request
 import zipfile
-from queue import Queue
 from typing import Any, Iterable, List, Optional
 
-import torch
-from torch.utils.data import Dataset
 from torch.utils.model_zoo import tqdm
 
-from torchaudio._internal.module_utils import deprecated
-
 
 def stream_url(url: str,
                start_byte: Optional[int] = None,
@@ -203,82 +197,3 @@
         pass
 
     raise NotImplementedError("We currently only support tar.gz, tgz, and zip achives.")
-
-
-class _DiskCache(Dataset):
-    """
-    Wrap a dataset so that, whenever a new item is returned, it is saved to disk.
-    """
-
-    def __init__(self, dataset: Dataset, location: str = ".cached") -> None:
-        self.dataset = dataset
-        self.location = location
-
-        self._id = id(self)
-        self._cache: List = [None] * len(dataset)
-
-    def __getitem__(self, n: int) -> Any:
-        if self._cache[n]:
-            f = self._cache[n]
-            return torch.load(f)
-
-        f = str(self._id) + "-" + str(n)
-        f = os.path.join(self.location, f)
-        item = self.dataset[n]
-
-        self._cache[n] = f
-        os.makedirs(self.location, exist_ok=True)
-        torch.save(item, f)
-
-        return item
-
-    def __len__(self) -> int:
-        return len(self.dataset)
-
-
-@deprecated('', version='0.11')
-def diskcache_iterator(dataset: Dataset, location: str = ".cached") -> Dataset:
-    return _DiskCache(dataset, location)
-
-
-class _ThreadedIterator(threading.Thread):
-    """
-    Prefetch the next queue_length items from iterator in a background thread.
-
-    Example:
-    >> for i in bg_iterator(range(10)):
-    >>     print(i)
-    """
-
-    class _End:
-        pass
-
-    def __init__(self, generator: Iterable, maxsize: int) -> None:
-        threading.Thread.__init__(self)
-        self.queue: Queue = Queue(maxsize)
-        self.generator = generator
-        self.daemon = True
-        self.start()
-
-    def run(self) -> None:
-        for item in self.generator:
-            self.queue.put(item)
-        self.queue.put(self._End)
-
-    def __iter__(self) -> Any:
-        return self
-
-    def __next__(self) -> Any:
-        next_item = self.queue.get()
-        if next_item == self._End:
-            raise StopIteration
-        return next_item
-
-    # Required for Python 2.7 compatibility
-    def next(self) -> Any:
-        return self.__next__()
-
-
-@deprecated('', version='0.11')
-def bg_iterator(iterable: Iterable, maxsize: int) -> Any:
-    return _ThreadedIterator(iterable, maxsize=maxsize)

{"golden_diff": "diff --git a/torchaudio/datasets/__init__.py b/torchaudio/datasets/__init__.py\n--- a/torchaudio/datasets/__init__.py\n+++ b/torchaudio/datasets/__init__.py\n@@ -1,7 +1,6 @@\n from .commonvoice import COMMONVOICE\n from .librispeech import LIBRISPEECH\n from .speechcommands import SPEECHCOMMANDS\n-from .utils import bg_iterator, diskcache_iterator\n from .vctk import VCTK_092\n from .gtzan import GTZAN\n from .yesno import YESNO\n@@ -23,7 +22,5 @@\n     \"CMUARCTIC\",\n     \"CMUDict\",\n     \"LIBRITTS\",\n-    \"diskcache_iterator\",\n-    \"bg_iterator\",\n     \"TEDLIUM\",\n ]\ndiff --git a/torchaudio/datasets/utils.py b/torchaudio/datasets/utils.py\n--- a/torchaudio/datasets/utils.py\n+++ b/torchaudio/datasets/utils.py\n@@ -2,19 +2,13 @@\n import logging\n import os\n import tarfile\n-import threading\n import urllib\n import urllib.request\n import zipfile\n-from queue import Queue\n from typing import Any, Iterable, List, Optional\n \n-import torch\n-from torch.utils.data import Dataset\n from torch.utils.model_zoo import tqdm\n \n-from torchaudio._internal.module_utils import deprecated\n-\n \n def stream_url(url: str,\n                start_byte: Optional[int] = None,\n@@ -203,82 +197,3 @@\n         pass\n \n     raise NotImplementedError(\"We currently only support tar.gz, tgz, and zip achives.\")\n-\n-\n-class _DiskCache(Dataset):\n-    \"\"\"\n-    Wrap a dataset so that, whenever a new item is returned, it is saved to disk.\n-    \"\"\"\n-\n-    def __init__(self, dataset: Dataset, location: str = \".cached\") -> None:\n-        self.dataset = dataset\n-        self.location = location\n-\n-        self._id = id(self)\n-        self._cache: List = [None] * len(dataset)\n-\n-    def __getitem__(self, n: int) -> Any:\n-        if self._cache[n]:\n-            f = self._cache[n]\n-            return torch.load(f)\n-\n-        f = str(self._id) + \"-\" + str(n)\n-        f = os.path.join(self.location, f)\n-        item = self.dataset[n]\n-\n-        self._cache[n] = f\n-        os.makedirs(self.location, exist_ok=True)\n-        torch.save(item, f)\n-\n-        return item\n-\n-    def __len__(self) -> int:\n-        return len(self.dataset)\n-\n-\n-@deprecated('', version='0.11')\n-def diskcache_iterator(dataset: Dataset, location: str = \".cached\") -> Dataset:\n-    return _DiskCache(dataset, location)\n-\n-\n-class _ThreadedIterator(threading.Thread):\n-    \"\"\"\n-    Prefetch the next queue_length items from iterator in a background thread.\n-\n-    Example:\n-    >> for i in bg_iterator(range(10)):\n-    >>     print(i)\n-    \"\"\"\n-\n-    class _End:\n-        pass\n-\n-    def __init__(self, generator: Iterable, maxsize: int) -> None:\n-        threading.Thread.__init__(self)\n-        self.queue: Queue = Queue(maxsize)\n-        self.generator = generator\n-        self.daemon = True\n-        self.start()\n-\n-    def run(self) -> None:\n-        for item in self.generator:\n-            self.queue.put(item)\n-        self.queue.put(self._End)\n-\n-    def __iter__(self) -> Any:\n-        return self\n-\n-    def __next__(self) -> Any:\n-        next_item = self.queue.get()\n-        if next_item == self._End:\n-            raise StopIteration\n-        return next_item\n-\n-    # Required for Python 2.7 compatibility\n-    def next(self) -> Any:\n-        return self.__next__()\n-\n-\n-@deprecated('', version='0.11')\n-def bg_iterator(iterable: Iterable, maxsize: int) -> Any:\n-    return _ThreadedIterator(iterable, maxsize=maxsize)\n", "issue": "doc for bg_iterator et al\nThe [documentation](https://pytorch.org/audio/datasets.html) for torchaudio does not include `bg_iterator` and `disk_cache_iterator` from [code](https://github.com/pytorch/audio/blob/master/torchaudio/datasets/utils.py). Thanks to [comment](https://discuss.pytorch.org/t/audio-dataset-load-large-file-into-memory-in-background/85943) to point this out :)\n", "before_files": [{"content": "from .commonvoice import COMMONVOICE\nfrom .librispeech import LIBRISPEECH\nfrom .speechcommands import SPEECHCOMMANDS\nfrom .utils import bg_iterator, diskcache_iterator\nfrom .vctk import VCTK_092\nfrom .gtzan import GTZAN\nfrom .yesno import YESNO\nfrom .ljspeech import LJSPEECH\nfrom .cmuarctic import CMUARCTIC\nfrom .cmudict import CMUDict\nfrom .libritts import LIBRITTS\nfrom .tedlium import TEDLIUM\n\n\n__all__ = [\n    \"COMMONVOICE\",\n    \"LIBRISPEECH\",\n    \"SPEECHCOMMANDS\",\n    \"VCTK_092\",\n    \"YESNO\",\n    \"LJSPEECH\",\n    \"GTZAN\",\n    \"CMUARCTIC\",\n    \"CMUDict\",\n    \"LIBRITTS\",\n    \"diskcache_iterator\",\n    \"bg_iterator\",\n    \"TEDLIUM\",\n]\n", "path": "torchaudio/datasets/__init__.py"}, {"content": "import hashlib\nimport logging\nimport os\nimport tarfile\nimport threading\nimport urllib\nimport urllib.request\nimport zipfile\nfrom queue import Queue\nfrom typing import Any, Iterable, List, Optional\n\nimport torch\nfrom torch.utils.data import Dataset\nfrom torch.utils.model_zoo import tqdm\n\nfrom torchaudio._internal.module_utils import deprecated\n\n\ndef stream_url(url: str,\n               start_byte: Optional[int] = None,\n               block_size: int = 32 * 1024,\n               progress_bar: bool = True) -> Iterable:\n    \"\"\"Stream url by chunk\n\n    Args:\n        url (str): Url.\n        start_byte (int or None, optional): Start streaming at that point (Default: ``None``).\n        block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n    \"\"\"\n\n    # If we already have the whole file, there is no need to download it again\n    req = urllib.request.Request(url, method=\"HEAD\")\n    with urllib.request.urlopen(req) as response:\n        url_size = int(response.info().get(\"Content-Length\", -1))\n    if url_size == start_byte:\n        return\n\n    req = urllib.request.Request(url)\n    if start_byte:\n        req.headers[\"Range\"] = \"bytes={}-\".format(start_byte)\n\n    with urllib.request.urlopen(req) as upointer, tqdm(\n            unit=\"B\",\n            unit_scale=True,\n            unit_divisor=1024,\n            total=url_size,\n            disable=not progress_bar,\n    ) as pbar:\n\n        num_bytes = 0\n        while True:\n            chunk = upointer.read(block_size)\n            if not chunk:\n                break\n            yield chunk\n            num_bytes += len(chunk)\n            pbar.update(len(chunk))\n\n\ndef download_url(url: str,\n                 download_folder: str,\n                 filename: Optional[str] = None,\n                 hash_value: Optional[str] = None,\n                 hash_type: str = \"sha256\",\n                 progress_bar: bool = True,\n                 resume: bool = False) -> None:\n    \"\"\"Download file to disk.\n\n    Args:\n        url (str): Url.\n        download_folder (str): Folder to download file.\n        filename (str or None, optional): Name of downloaded file. If None, it is inferred from the url\n            (Default: ``None``).\n        hash_value (str or None, optional): Hash for url (Default: ``None``).\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n        resume (bool, optional): Enable resuming download (Default: ``False``).\n    \"\"\"\n\n    req = urllib.request.Request(url, method=\"HEAD\")\n    req_info = urllib.request.urlopen(req).info()\n\n    # Detect filename\n    filename = filename or req_info.get_filename() or os.path.basename(url)\n    filepath = os.path.join(download_folder, filename)\n    if resume and os.path.exists(filepath):\n        mode = \"ab\"\n        local_size: Optional[int] = os.path.getsize(filepath)\n\n    elif not resume and os.path.exists(filepath):\n        raise RuntimeError(\n            \"{} already exists. Delete the file manually and retry.\".format(filepath)\n        )\n    else:\n        mode = \"wb\"\n        local_size = None\n\n    if hash_value and local_size == int(req_info.get(\"Content-Length\", -1)):\n        with open(filepath, \"rb\") as file_obj:\n            if validate_file(file_obj, hash_value, hash_type):\n                return\n        raise RuntimeError(\n            \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                filepath\n            )\n        )\n\n    with open(filepath, mode) as fpointer:\n        for chunk in stream_url(url, start_byte=local_size, progress_bar=progress_bar):\n            fpointer.write(chunk)\n\n    with open(filepath, \"rb\") as file_obj:\n        if hash_value and not validate_file(file_obj, hash_value, hash_type):\n            raise RuntimeError(\n                \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                    filepath\n                )\n            )\n\n\ndef validate_file(file_obj: Any, hash_value: str, hash_type: str = \"sha256\") -> bool:\n    \"\"\"Validate a given file object with its hash.\n\n    Args:\n        file_obj: File object to read from.\n        hash_value (str): Hash for url.\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n\n    Returns:\n        bool: return True if its a valid file, else False.\n    \"\"\"\n\n    if hash_type == \"sha256\":\n        hash_func = hashlib.sha256()\n    elif hash_type == \"md5\":\n        hash_func = hashlib.md5()\n    else:\n        raise ValueError\n\n    while True:\n        # Read by chunk to avoid filling memory\n        chunk = file_obj.read(1024 ** 2)\n        if not chunk:\n            break\n        hash_func.update(chunk)\n\n    return hash_func.hexdigest() == hash_value\n\n\ndef extract_archive(from_path: str, to_path: Optional[str] = None, overwrite: bool = False) -> List[str]:\n    \"\"\"Extract archive.\n    Args:\n        from_path (str): the path of the archive.\n        to_path (str or None, optional): the root path of the extraced files (directory of from_path)\n            (Default: ``None``)\n        overwrite (bool, optional): overwrite existing files (Default: ``False``)\n\n    Returns:\n        list: List of paths to extracted files even if not overwritten.\n\n    Examples:\n        >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'\n        >>> from_path = './validation.tar.gz'\n        >>> to_path = './'\n        >>> torchaudio.datasets.utils.download_from_url(url, from_path)\n        >>> torchaudio.datasets.utils.extract_archive(from_path, to_path)\n    \"\"\"\n\n    if to_path is None:\n        to_path = os.path.dirname(from_path)\n\n    try:\n        with tarfile.open(from_path, \"r\") as tar:\n            logging.info(\"Opened tar file {}.\".format(from_path))\n            files = []\n            for file_ in tar:  # type: Any\n                file_path = os.path.join(to_path, file_.name)\n                if file_.isfile():\n                    files.append(file_path)\n                    if os.path.exists(file_path):\n                        logging.info(\"{} already extracted.\".format(file_path))\n                        if not overwrite:\n                            continue\n                tar.extract(file_, to_path)\n            return files\n    except tarfile.ReadError:\n        pass\n\n    try:\n        with zipfile.ZipFile(from_path, \"r\") as zfile:\n            logging.info(\"Opened zip file {}.\".format(from_path))\n            files = zfile.namelist()\n            for file_ in files:\n                file_path = os.path.join(to_path, file_)\n                if os.path.exists(file_path):\n                    logging.info(\"{} already extracted.\".format(file_path))\n                    if not overwrite:\n                        continue\n                zfile.extract(file_, to_path)\n        return files\n    except zipfile.BadZipFile:\n        pass\n\n    raise NotImplementedError(\"We currently only support tar.gz, tgz, and zip achives.\")\n\n\nclass _DiskCache(Dataset):\n    \"\"\"\n    Wrap a dataset so that, whenever a new item is returned, it is saved to disk.\n    \"\"\"\n\n    def __init__(self, dataset: Dataset, location: str = \".cached\") -> None:\n        self.dataset = dataset\n        self.location = location\n\n        self._id = id(self)\n        self._cache: List = [None] * len(dataset)\n\n    def __getitem__(self, n: int) -> Any:\n        if self._cache[n]:\n            f = self._cache[n]\n            return torch.load(f)\n\n        f = str(self._id) + \"-\" + str(n)\n        f = os.path.join(self.location, f)\n        item = self.dataset[n]\n\n        self._cache[n] = f\n        os.makedirs(self.location, exist_ok=True)\n        torch.save(item, f)\n\n        return item\n\n    def __len__(self) -> int:\n        return len(self.dataset)\n\n\n@deprecated('', version='0.11')\ndef diskcache_iterator(dataset: Dataset, location: str = \".cached\") -> Dataset:\n    return _DiskCache(dataset, location)\n\n\nclass _ThreadedIterator(threading.Thread):\n    \"\"\"\n    Prefetch the next queue_length items from iterator in a background thread.\n\n    Example:\n    >> for i in bg_iterator(range(10)):\n    >>     print(i)\n    \"\"\"\n\n    class _End:\n        pass\n\n    def __init__(self, generator: Iterable, maxsize: int) -> None:\n        threading.Thread.__init__(self)\n        self.queue: Queue = Queue(maxsize)\n        self.generator = generator\n        self.daemon = True\n        self.start()\n\n    def run(self) -> None:\n        for item in self.generator:\n            self.queue.put(item)\n        self.queue.put(self._End)\n\n    def __iter__(self) -> Any:\n        return self\n\n    def __next__(self) -> Any:\n        next_item = self.queue.get()\n        if next_item == self._End:\n            raise StopIteration\n        return next_item\n\n    # Required for Python 2.7 compatibility\n    def next(self) -> Any:\n        return self.__next__()\n\n\n@deprecated('', version='0.11')\ndef bg_iterator(iterable: Iterable, maxsize: int) -> Any:\n    return _ThreadedIterator(iterable, maxsize=maxsize)\n", "path": "torchaudio/datasets/utils.py"}], "after_files": [{"content": "from .commonvoice import COMMONVOICE\nfrom .librispeech import LIBRISPEECH\nfrom .speechcommands import SPEECHCOMMANDS\nfrom .vctk import VCTK_092\nfrom .gtzan import GTZAN\nfrom .yesno import YESNO\nfrom .ljspeech import LJSPEECH\nfrom .cmuarctic import CMUARCTIC\nfrom .cmudict import CMUDict\nfrom .libritts import LIBRITTS\nfrom .tedlium import TEDLIUM\n\n\n__all__ = [\n    \"COMMONVOICE\",\n    \"LIBRISPEECH\",\n    \"SPEECHCOMMANDS\",\n    \"VCTK_092\",\n    \"YESNO\",\n    \"LJSPEECH\",\n    \"GTZAN\",\n    \"CMUARCTIC\",\n    \"CMUDict\",\n    \"LIBRITTS\",\n    \"TEDLIUM\",\n]\n", "path": "torchaudio/datasets/__init__.py"}, {"content": "import hashlib\nimport logging\nimport os\nimport tarfile\nimport urllib\nimport urllib.request\nimport zipfile\nfrom typing import Any, Iterable, List, Optional\n\nfrom torch.utils.model_zoo import tqdm\n\n\ndef stream_url(url: str,\n               start_byte: Optional[int] = None,\n               block_size: int = 32 * 1024,\n               progress_bar: bool = True) -> Iterable:\n    \"\"\"Stream url by chunk\n\n    Args:\n        url (str): Url.\n        start_byte (int or None, optional): Start streaming at that point (Default: ``None``).\n        block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n    \"\"\"\n\n    # If we already have the whole file, there is no need to download it again\n    req = urllib.request.Request(url, method=\"HEAD\")\n    with urllib.request.urlopen(req) as response:\n        url_size = int(response.info().get(\"Content-Length\", -1))\n    if url_size == start_byte:\n        return\n\n    req = urllib.request.Request(url)\n    if start_byte:\n        req.headers[\"Range\"] = \"bytes={}-\".format(start_byte)\n\n    with urllib.request.urlopen(req) as upointer, tqdm(\n            unit=\"B\",\n            unit_scale=True,\n            unit_divisor=1024,\n            total=url_size,\n            disable=not progress_bar,\n    ) as pbar:\n\n        num_bytes = 0\n        while True:\n            chunk = upointer.read(block_size)\n            if not chunk:\n                break\n            yield chunk\n            num_bytes += len(chunk)\n            pbar.update(len(chunk))\n\n\ndef download_url(url: str,\n                 download_folder: str,\n                 filename: Optional[str] = None,\n                 hash_value: Optional[str] = None,\n                 hash_type: str = \"sha256\",\n                 progress_bar: bool = True,\n                 resume: bool = False) -> None:\n    \"\"\"Download file to disk.\n\n    Args:\n        url (str): Url.\n        download_folder (str): Folder to download file.\n        filename (str or None, optional): Name of downloaded file. If None, it is inferred from the url\n            (Default: ``None``).\n        hash_value (str or None, optional): Hash for url (Default: ``None``).\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n        resume (bool, optional): Enable resuming download (Default: ``False``).\n    \"\"\"\n\n    req = urllib.request.Request(url, method=\"HEAD\")\n    req_info = urllib.request.urlopen(req).info()\n\n    # Detect filename\n    filename = filename or req_info.get_filename() or os.path.basename(url)\n    filepath = os.path.join(download_folder, filename)\n    if resume and os.path.exists(filepath):\n        mode = \"ab\"\n        local_size: Optional[int] = os.path.getsize(filepath)\n\n    elif not resume and os.path.exists(filepath):\n        raise RuntimeError(\n            \"{} already exists. Delete the file manually and retry.\".format(filepath)\n        )\n    else:\n        mode = \"wb\"\n        local_size = None\n\n    if hash_value and local_size == int(req_info.get(\"Content-Length\", -1)):\n        with open(filepath, \"rb\") as file_obj:\n            if validate_file(file_obj, hash_value, hash_type):\n                return\n        raise RuntimeError(\n            \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                filepath\n            )\n        )\n\n    with open(filepath, mode) as fpointer:\n        for chunk in stream_url(url, start_byte=local_size, progress_bar=progress_bar):\n            fpointer.write(chunk)\n\n    with open(filepath, \"rb\") as file_obj:\n        if hash_value and not validate_file(file_obj, hash_value, hash_type):\n            raise RuntimeError(\n                \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                    filepath\n                )\n            )\n\n\ndef validate_file(file_obj: Any, hash_value: str, hash_type: str = \"sha256\") -> bool:\n    \"\"\"Validate a given file object with its hash.\n\n    Args:\n        file_obj: File object to read from.\n        hash_value (str): Hash for url.\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n\n    Returns:\n        bool: return True if its a valid file, else False.\n    \"\"\"\n\n    if hash_type == \"sha256\":\n        hash_func = hashlib.sha256()\n    elif hash_type == \"md5\":\n        hash_func = hashlib.md5()\n    else:\n        raise ValueError\n\n    while True:\n        # Read by chunk to avoid filling memory\n        chunk = file_obj.read(1024 ** 2)\n        if not chunk:\n            break\n        hash_func.update(chunk)\n\n    return hash_func.hexdigest() == hash_value\n\n\ndef extract_archive(from_path: str, to_path: Optional[str] = None, overwrite: bool = False) -> List[str]:\n    \"\"\"Extract archive.\n    Args:\n        from_path (str): the path of the archive.\n        to_path (str or None, optional): the root path of the extraced files (directory of from_path)\n            (Default: ``None``)\n        overwrite (bool, optional): overwrite existing files (Default: ``False``)\n\n    Returns:\n        list: List of paths to extracted files even if not overwritten.\n\n    Examples:\n        >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'\n        >>> from_path = './validation.tar.gz'\n        >>> to_path = './'\n        >>> torchaudio.datasets.utils.download_from_url(url, from_path)\n        >>> torchaudio.datasets.utils.extract_archive(from_path, to_path)\n    \"\"\"\n\n    if to_path is None:\n        to_path = os.path.dirname(from_path)\n\n    try:\n        with tarfile.open(from_path, \"r\") as tar:\n            logging.info(\"Opened tar file {}.\".format(from_path))\n            files = []\n            for file_ in tar:  # type: Any\n                file_path = os.path.join(to_path, file_.name)\n                if file_.isfile():\n                    files.append(file_path)\n                    if os.path.exists(file_path):\n                        logging.info(\"{} already extracted.\".format(file_path))\n                        if not overwrite:\n                            continue\n                tar.extract(file_, to_path)\n            return files\n    except tarfile.ReadError:\n        pass\n\n    try:\n        with zipfile.ZipFile(from_path, \"r\") as zfile:\n            logging.info(\"Opened zip file {}.\".format(from_path))\n            files = zfile.namelist()\n            for file_ in files:\n                file_path = os.path.join(to_path, file_)\n                if os.path.exists(file_path):\n                    logging.info(\"{} already extracted.\".format(file_path))\n                    if not overwrite:\n                        continue\n                zfile.extract(file_, to_path)\n        return files\n    except zipfile.BadZipFile:\n        pass\n\n    raise NotImplementedError(\"We currently only support tar.gz, tgz, and zip achives.\")\n", "path": "torchaudio/datasets/utils.py"}]}