We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Dataset traversal cannot be deterministic
[Many dataset implementations use `walk_files` function](https://github.com/pytorch/audio/search?q=walk_files&unscoped_q=walk_files) which uses `os.walk` which does not give us choice to make directory traversal in deterministic order.

This makes it harder to test dataset and causes issue on reproducibility. Dataset should have an option to be deterministic.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchaudio/datasets/utils.py`
Content:
```
1 import csv
2 import errno
3 import hashlib
4 import logging
5 import os
6 import sys
7 import tarfile
8 import threading
9 import zipfile
10 from _io import TextIOWrapper
11 from queue import Queue
12 from typing import Any, Iterable, List, Optional, Tuple, Union
13 
14 import torch
15 import urllib
16 import urllib.request
17 from torch.utils.data import Dataset
18 from torch.utils.model_zoo import tqdm
19 
20 
21 def unicode_csv_reader(unicode_csv_data: TextIOWrapper, **kwargs: Any) -> Any:
22     r"""Since the standard csv library does not handle unicode in Python 2, we need a wrapper.
23     Borrowed and slightly modified from the Python docs:
24     https://docs.python.org/2/library/csv.html#csv-examples
25     Args:
26         unicode_csv_data (TextIOWrapper): unicode csv data (see example below)
27 
28     Examples:
29         >>> from torchaudio.datasets.utils import unicode_csv_reader
30         >>> import io
31         >>> with io.open(data_path, encoding="utf8") as f:
32         >>>     reader = unicode_csv_reader(f)
33     """
34 
35     # Fix field larger than field limit error
36     maxInt = sys.maxsize
37     while True:
38         # decrease the maxInt value by factor 10
39         # as long as the OverflowError occurs.
40         try:
41             csv.field_size_limit(maxInt)
42             break
43         except OverflowError:
44             maxInt = int(maxInt / 10)
45     csv.field_size_limit(maxInt)
46 
47     for line in csv.reader(unicode_csv_data, **kwargs):
48         yield line
49 
50 
51 def makedir_exist_ok(dirpath: str) -> None:
52     """
53     Python2 support for os.makedirs(.., exist_ok=True)
54     """
55     try:
56         os.makedirs(dirpath)
57     except OSError as e:
58         if e.errno == errno.EEXIST:
59             pass
60         else:
61             raise
62 
63 
64 def stream_url(url: str,
65                start_byte: Optional[int] = None,
66                block_size: int = 32 * 1024,
67                progress_bar: bool = True) -> Iterable:
68     """Stream url by chunk
69 
70     Args:
71         url (str): Url.
72         start_byte (int, optional): Start streaming at that point (Default: ``None``).
73         block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``).
74         progress_bar (bool, optional): Display a progress bar (Default: ``True``).
75     """
76 
77     # If we already have the whole file, there is no need to download it again
78     req = urllib.request.Request(url, method="HEAD")
79     url_size = int(urllib.request.urlopen(req).info().get("Content-Length", -1))
80     if url_size == start_byte:
81         return
82 
83     req = urllib.request.Request(url)
84     if start_byte:
85         req.headers["Range"] = "bytes={}-".format(start_byte)
86 
87     with urllib.request.urlopen(req) as upointer, tqdm(
88         unit="B",
89         unit_scale=True,
90         unit_divisor=1024,
91         total=url_size,
92         disable=not progress_bar,
93     ) as pbar:
94 
95         num_bytes = 0
96         while True:
97             chunk = upointer.read(block_size)
98             if not chunk:
99                 break
100             yield chunk
101             num_bytes += len(chunk)
102             pbar.update(len(chunk))
103 
104 
105 def download_url(url: str,
106                  download_folder: str,
107                  filename: Optional[str] = None,
108                  hash_value: Optional[str] = None,
109                  hash_type: str = "sha256",
110                  progress_bar: bool = True,
111                  resume: bool = False) -> None:
112     """Download file to disk.
113 
114     Args:
115         url (str): Url.
116         download_folder (str): Folder to download file.
117         filename (str, optional): Name of downloaded file. If None, it is inferred from the url (Default: ``None``).
118         hash_value (str, optional): Hash for url (Default: ``None``).
119         hash_type (str, optional): Hash type, among "sha256" and "md5" (Default: ``"sha256"``).
120         progress_bar (bool, optional): Display a progress bar (Default: ``True``).
121         resume (bool, optional): Enable resuming download (Default: ``False``).
122     """
123 
124     req = urllib.request.Request(url, method="HEAD")
125     req_info = urllib.request.urlopen(req).info()
126 
127     # Detect filename
128     filename = filename or req_info.get_filename() or os.path.basename(url)
129     filepath = os.path.join(download_folder, filename)
130     if resume and os.path.exists(filepath):
131         mode = "ab"
132         local_size: Optional[int] = os.path.getsize(filepath)
133 
134     elif not resume and os.path.exists(filepath):
135         raise RuntimeError(
136             "{} already exists. Delete the file manually and retry.".format(filepath)
137         )
138     else:
139         mode = "wb"
140         local_size = None
141 
142     if hash_value and local_size == int(req_info.get("Content-Length", -1)):
143         with open(filepath, "rb") as file_obj:
144             if validate_file(file_obj, hash_value, hash_type):
145                 return
146         raise RuntimeError(
147             "The hash of {} does not match. Delete the file manually and retry.".format(
148                 filepath
149             )
150         )
151 
152     with open(filepath, mode) as fpointer:
153         for chunk in stream_url(url, start_byte=local_size, progress_bar=progress_bar):
154             fpointer.write(chunk)
155 
156     with open(filepath, "rb") as file_obj:
157         if hash_value and not validate_file(file_obj, hash_value, hash_type):
158             raise RuntimeError(
159                 "The hash of {} does not match. Delete the file manually and retry.".format(
160                     filepath
161                 )
162             )
163 
164 
165 def validate_file(file_obj: Any, hash_value: str, hash_type: str = "sha256") -> bool:
166     """Validate a given file object with its hash.
167 
168     Args:
169         file_obj: File object to read from.
170         hash_value (str): Hash for url.
171         hash_type (str, optional): Hash type, among "sha256" and "md5" (Default: ``"sha256"``).
172 
173     Returns:
174         bool: return True if its a valid file, else False.
175     """
176 
177     if hash_type == "sha256":
178         hash_func = hashlib.sha256()
179     elif hash_type == "md5":
180         hash_func = hashlib.md5()
181     else:
182         raise ValueError
183 
184     while True:
185         # Read by chunk to avoid filling memory
186         chunk = file_obj.read(1024 ** 2)
187         if not chunk:
188             break
189         hash_func.update(chunk)
190 
191     return hash_func.hexdigest() == hash_value
192 
193 
194 def extract_archive(from_path: str, to_path: Optional[str] = None, overwrite: bool = False) -> List[str]:
195     """Extract archive.
196     Args:
197         from_path (str): the path of the archive.
198         to_path (str, optional): the root path of the extraced files (directory of from_path) (Default: ``None``)
199         overwrite (bool, optional): overwrite existing files (Default: ``False``)
200 
201     Returns:
202         list: List of paths to extracted files even if not overwritten.
203 
204     Examples:
205         >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'
206         >>> from_path = './validation.tar.gz'
207         >>> to_path = './'
208         >>> torchaudio.datasets.utils.download_from_url(url, from_path)
209         >>> torchaudio.datasets.utils.extract_archive(from_path, to_path)
210     """
211 
212     if to_path is None:
213         to_path = os.path.dirname(from_path)
214 
215     try:
216         with tarfile.open(from_path, "r") as tar:
217             logging.info("Opened tar file {}.".format(from_path))
218             files = []
219             for file_ in tar:  # type: Any
220                 file_path = os.path.join(to_path, file_.name)
221                 if file_.isfile():
222                     files.append(file_path)
223                     if os.path.exists(file_path):
224                         logging.info("{} already extracted.".format(file_path))
225                         if not overwrite:
226                             continue
227                 tar.extract(file_, to_path)
228             return files
229     except tarfile.ReadError:
230         pass
231 
232     try:
233         with zipfile.ZipFile(from_path, "r") as zfile:
234             logging.info("Opened zip file {}.".format(from_path))
235             files = zfile.namelist()
236             for file_ in files:
237                 file_path = os.path.join(to_path, file_)
238                 if os.path.exists(file_path):
239                     logging.info("{} already extracted.".format(file_path))
240                     if not overwrite:
241                         continue
242                 zfile.extract(file_, to_path)
243         return files
244     except zipfile.BadZipFile:
245         pass
246 
247     raise NotImplementedError("We currently only support tar.gz, tgz, and zip achives.")
248 
249 
250 def walk_files(root: str,
251                suffix: Union[str, Tuple[str]],
252                prefix: bool = False,
253                remove_suffix: bool = False) -> Iterable[str]:
254     """List recursively all files ending with a suffix at a given root
255     Args:
256         root (str): Path to directory whose folders need to be listed
257         suffix (str or tuple): Suffix of the files to match, e.g. '.png' or ('.jpg', '.png').
258             It uses the Python "str.endswith" method and is passed directly
259         prefix (bool, optional): If true, prepends the full path to each result, otherwise
260             only returns the name of the files found (Default: ``False``)
261         remove_suffix (bool, optional): If true, removes the suffix to each result defined in suffix,
262             otherwise will return the result as found (Default: ``False``).
263     """
264 
265     root = os.path.expanduser(root)
266 
267     for dirpath, _, files in os.walk(root):
268         for f in files:
269             if f.endswith(suffix):
270 
271                 if remove_suffix:
272                     f = f[: -len(suffix)]
273 
274                 if prefix:
275                     f = os.path.join(dirpath, f)
276 
277                 yield f
278 
279 
280 class _DiskCache(Dataset):
281     """
282     Wrap a dataset so that, whenever a new item is returned, it is saved to disk.
283     """
284 
285     def __init__(self, dataset: Dataset, location: str = ".cached") -> None:
286         self.dataset = dataset
287         self.location = location
288 
289         self._id = id(self)
290         self._cache: List = [None] * len(dataset)
291 
292     def __getitem__(self, n: int) -> Any:
293         if self._cache[n]:
294             f = self._cache[n]
295             return torch.load(f)
296 
297         f = str(self._id) + "-" + str(n)
298         f = os.path.join(self.location, f)
299         item = self.dataset[n]
300 
301         self._cache[n] = f
302         makedir_exist_ok(self.location)
303         torch.save(item, f)
304 
305         return item
306 
307     def __len__(self) -> int:
308         return len(self.dataset)
309 
310 
311 def diskcache_iterator(dataset: Dataset, location: str = ".cached") -> Dataset:
312     return _DiskCache(dataset, location)
313 
314 
315 class _ThreadedIterator(threading.Thread):
316     """
317     Prefetch the next queue_length items from iterator in a background thread.
318 
319     Example:
320     >> for i in bg_iterator(range(10)):
321     >>     print(i)
322     """
323 
324     class _End:
325         pass
326 
327     def __init__(self, generator: Iterable, maxsize: int) -> None:
328         threading.Thread.__init__(self)
329         self.queue: Queue = Queue(maxsize)
330         self.generator = generator
331         self.daemon = True
332         self.start()
333 
334     def run(self) -> None:
335         for item in self.generator:
336             self.queue.put(item)
337         self.queue.put(self._End)
338 
339     def __iter__(self) -> Any:
340         return self
341 
342     def __next__(self) -> Any:
343         next_item = self.queue.get()
344         if next_item == self._End:
345             raise StopIteration
346         return next_item
347 
348     # Required for Python 2.7 compatibility
349     def next(self) -> Any:
350         return self.__next__()
351 
352 
353 def bg_iterator(iterable: Iterable, maxsize: int) -> Any:
354     return _ThreadedIterator(iterable, maxsize=maxsize)
355
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchaudio/datasets/utils.py b/torchaudio/datasets/utils.py
--- a/torchaudio/datasets/utils.py
+++ b/torchaudio/datasets/utils.py
@@ -264,7 +264,13 @@
 
     root = os.path.expanduser(root)
 
-    for dirpath, _, files in os.walk(root):
+    for dirpath, dirs, files in os.walk(root):
+        dirs.sort()
+        # `dirs` is the list used in os.walk function and by sorting it in-place here, we change the
+        # behavior of os.walk to traverse sub directory alphabetically
+        # see also
+        # https://stackoverflow.com/questions/6670029/can-i-force-python3s-os-walk-to-visit-directories-in-alphabetical-order-how#comment71993866_6670926
+        files.sort()
         for f in files:
             if f.endswith(suffix):

{"golden_diff": "diff --git a/torchaudio/datasets/utils.py b/torchaudio/datasets/utils.py\n--- a/torchaudio/datasets/utils.py\n+++ b/torchaudio/datasets/utils.py\n@@ -264,7 +264,13 @@\n \n     root = os.path.expanduser(root)\n \n-    for dirpath, _, files in os.walk(root):\n+    for dirpath, dirs, files in os.walk(root):\n+        dirs.sort()\n+        # `dirs` is the list used in os.walk function and by sorting it in-place here, we change the\n+        # behavior of os.walk to traverse sub directory alphabetically\n+        # see also\n+        # https://stackoverflow.com/questions/6670029/can-i-force-python3s-os-walk-to-visit-directories-in-alphabetical-order-how#comment71993866_6670926\n+        files.sort()\n         for f in files:\n             if f.endswith(suffix):\n", "issue": "Dataset traversal cannot be deterministic\n[Many dataset implementations use `walk_files` function](https://github.com/pytorch/audio/search?q=walk_files&unscoped_q=walk_files) which uses `os.walk` which does not give us choice to make directory traversal in deterministic order.\r\n\r\nThis makes it harder to test dataset and causes issue on reproducibility. Dataset should have an option to be deterministic.\n", "before_files": [{"content": "import csv\nimport errno\nimport hashlib\nimport logging\nimport os\nimport sys\nimport tarfile\nimport threading\nimport zipfile\nfrom _io import TextIOWrapper\nfrom queue import Queue\nfrom typing import Any, Iterable, List, Optional, Tuple, Union\n\nimport torch\nimport urllib\nimport urllib.request\nfrom torch.utils.data import Dataset\nfrom torch.utils.model_zoo import tqdm\n\n\ndef unicode_csv_reader(unicode_csv_data: TextIOWrapper, **kwargs: Any) -> Any:\n    r\"\"\"Since the standard csv library does not handle unicode in Python 2, we need a wrapper.\n    Borrowed and slightly modified from the Python docs:\n    https://docs.python.org/2/library/csv.html#csv-examples\n    Args:\n        unicode_csv_data (TextIOWrapper): unicode csv data (see example below)\n\n    Examples:\n        >>> from torchaudio.datasets.utils import unicode_csv_reader\n        >>> import io\n        >>> with io.open(data_path, encoding=\"utf8\") as f:\n        >>>     reader = unicode_csv_reader(f)\n    \"\"\"\n\n    # Fix field larger than field limit error\n    maxInt = sys.maxsize\n    while True:\n        # decrease the maxInt value by factor 10\n        # as long as the OverflowError occurs.\n        try:\n            csv.field_size_limit(maxInt)\n            break\n        except OverflowError:\n            maxInt = int(maxInt / 10)\n    csv.field_size_limit(maxInt)\n\n    for line in csv.reader(unicode_csv_data, **kwargs):\n        yield line\n\n\ndef makedir_exist_ok(dirpath: str) -> None:\n    \"\"\"\n    Python2 support for os.makedirs(.., exist_ok=True)\n    \"\"\"\n    try:\n        os.makedirs(dirpath)\n    except OSError as e:\n        if e.errno == errno.EEXIST:\n            pass\n        else:\n            raise\n\n\ndef stream_url(url: str,\n               start_byte: Optional[int] = None,\n               block_size: int = 32 * 1024,\n               progress_bar: bool = True) -> Iterable:\n    \"\"\"Stream url by chunk\n\n    Args:\n        url (str): Url.\n        start_byte (int, optional): Start streaming at that point (Default: ``None``).\n        block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n    \"\"\"\n\n    # If we already have the whole file, there is no need to download it again\n    req = urllib.request.Request(url, method=\"HEAD\")\n    url_size = int(urllib.request.urlopen(req).info().get(\"Content-Length\", -1))\n    if url_size == start_byte:\n        return\n\n    req = urllib.request.Request(url)\n    if start_byte:\n        req.headers[\"Range\"] = \"bytes={}-\".format(start_byte)\n\n    with urllib.request.urlopen(req) as upointer, tqdm(\n        unit=\"B\",\n        unit_scale=True,\n        unit_divisor=1024,\n        total=url_size,\n        disable=not progress_bar,\n    ) as pbar:\n\n        num_bytes = 0\n        while True:\n            chunk = upointer.read(block_size)\n            if not chunk:\n                break\n            yield chunk\n            num_bytes += len(chunk)\n            pbar.update(len(chunk))\n\n\ndef download_url(url: str,\n                 download_folder: str,\n                 filename: Optional[str] = None,\n                 hash_value: Optional[str] = None,\n                 hash_type: str = \"sha256\",\n                 progress_bar: bool = True,\n                 resume: bool = False) -> None:\n    \"\"\"Download file to disk.\n\n    Args:\n        url (str): Url.\n        download_folder (str): Folder to download file.\n        filename (str, optional): Name of downloaded file. If None, it is inferred from the url (Default: ``None``).\n        hash_value (str, optional): Hash for url (Default: ``None``).\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n        resume (bool, optional): Enable resuming download (Default: ``False``).\n    \"\"\"\n\n    req = urllib.request.Request(url, method=\"HEAD\")\n    req_info = urllib.request.urlopen(req).info()\n\n    # Detect filename\n    filename = filename or req_info.get_filename() or os.path.basename(url)\n    filepath = os.path.join(download_folder, filename)\n    if resume and os.path.exists(filepath):\n        mode = \"ab\"\n        local_size: Optional[int] = os.path.getsize(filepath)\n\n    elif not resume and os.path.exists(filepath):\n        raise RuntimeError(\n            \"{} already exists. Delete the file manually and retry.\".format(filepath)\n        )\n    else:\n        mode = \"wb\"\n        local_size = None\n\n    if hash_value and local_size == int(req_info.get(\"Content-Length\", -1)):\n        with open(filepath, \"rb\") as file_obj:\n            if validate_file(file_obj, hash_value, hash_type):\n                return\n        raise RuntimeError(\n            \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                filepath\n            )\n        )\n\n    with open(filepath, mode) as fpointer:\n        for chunk in stream_url(url, start_byte=local_size, progress_bar=progress_bar):\n            fpointer.write(chunk)\n\n    with open(filepath, \"rb\") as file_obj:\n        if hash_value and not validate_file(file_obj, hash_value, hash_type):\n            raise RuntimeError(\n                \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                    filepath\n                )\n            )\n\n\ndef validate_file(file_obj: Any, hash_value: str, hash_type: str = \"sha256\") -> bool:\n    \"\"\"Validate a given file object with its hash.\n\n    Args:\n        file_obj: File object to read from.\n        hash_value (str): Hash for url.\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n\n    Returns:\n        bool: return True if its a valid file, else False.\n    \"\"\"\n\n    if hash_type == \"sha256\":\n        hash_func = hashlib.sha256()\n    elif hash_type == \"md5\":\n        hash_func = hashlib.md5()\n    else:\n        raise ValueError\n\n    while True:\n        # Read by chunk to avoid filling memory\n        chunk = file_obj.read(1024 ** 2)\n        if not chunk:\n            break\n        hash_func.update(chunk)\n\n    return hash_func.hexdigest() == hash_value\n\n\ndef extract_archive(from_path: str, to_path: Optional[str] = None, overwrite: bool = False) -> List[str]:\n    \"\"\"Extract archive.\n    Args:\n        from_path (str): the path of the archive.\n        to_path (str, optional): the root path of the extraced files (directory of from_path) (Default: ``None``)\n        overwrite (bool, optional): overwrite existing files (Default: ``False``)\n\n    Returns:\n        list: List of paths to extracted files even if not overwritten.\n\n    Examples:\n        >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'\n        >>> from_path = './validation.tar.gz'\n        >>> to_path = './'\n        >>> torchaudio.datasets.utils.download_from_url(url, from_path)\n        >>> torchaudio.datasets.utils.extract_archive(from_path, to_path)\n    \"\"\"\n\n    if to_path is None:\n        to_path = os.path.dirname(from_path)\n\n    try:\n        with tarfile.open(from_path, \"r\") as tar:\n            logging.info(\"Opened tar file {}.\".format(from_path))\n            files = []\n            for file_ in tar:  # type: Any\n                file_path = os.path.join(to_path, file_.name)\n                if file_.isfile():\n                    files.append(file_path)\n                    if os.path.exists(file_path):\n                        logging.info(\"{} already extracted.\".format(file_path))\n                        if not overwrite:\n                            continue\n                tar.extract(file_, to_path)\n            return files\n    except tarfile.ReadError:\n        pass\n\n    try:\n        with zipfile.ZipFile(from_path, \"r\") as zfile:\n            logging.info(\"Opened zip file {}.\".format(from_path))\n            files = zfile.namelist()\n            for file_ in files:\n                file_path = os.path.join(to_path, file_)\n                if os.path.exists(file_path):\n                    logging.info(\"{} already extracted.\".format(file_path))\n                    if not overwrite:\n                        continue\n                zfile.extract(file_, to_path)\n        return files\n    except zipfile.BadZipFile:\n        pass\n\n    raise NotImplementedError(\"We currently only support tar.gz, tgz, and zip achives.\")\n\n\ndef walk_files(root: str,\n               suffix: Union[str, Tuple[str]],\n               prefix: bool = False,\n               remove_suffix: bool = False) -> Iterable[str]:\n    \"\"\"List recursively all files ending with a suffix at a given root\n    Args:\n        root (str): Path to directory whose folders need to be listed\n        suffix (str or tuple): Suffix of the files to match, e.g. '.png' or ('.jpg', '.png').\n            It uses the Python \"str.endswith\" method and is passed directly\n        prefix (bool, optional): If true, prepends the full path to each result, otherwise\n            only returns the name of the files found (Default: ``False``)\n        remove_suffix (bool, optional): If true, removes the suffix to each result defined in suffix,\n            otherwise will return the result as found (Default: ``False``).\n    \"\"\"\n\n    root = os.path.expanduser(root)\n\n    for dirpath, _, files in os.walk(root):\n        for f in files:\n            if f.endswith(suffix):\n\n                if remove_suffix:\n                    f = f[: -len(suffix)]\n\n                if prefix:\n                    f = os.path.join(dirpath, f)\n\n                yield f\n\n\nclass _DiskCache(Dataset):\n    \"\"\"\n    Wrap a dataset so that, whenever a new item is returned, it is saved to disk.\n    \"\"\"\n\n    def __init__(self, dataset: Dataset, location: str = \".cached\") -> None:\n        self.dataset = dataset\n        self.location = location\n\n        self._id = id(self)\n        self._cache: List = [None] * len(dataset)\n\n    def __getitem__(self, n: int) -> Any:\n        if self._cache[n]:\n            f = self._cache[n]\n            return torch.load(f)\n\n        f = str(self._id) + \"-\" + str(n)\n        f = os.path.join(self.location, f)\n        item = self.dataset[n]\n\n        self._cache[n] = f\n        makedir_exist_ok(self.location)\n        torch.save(item, f)\n\n        return item\n\n    def __len__(self) -> int:\n        return len(self.dataset)\n\n\ndef diskcache_iterator(dataset: Dataset, location: str = \".cached\") -> Dataset:\n    return _DiskCache(dataset, location)\n\n\nclass _ThreadedIterator(threading.Thread):\n    \"\"\"\n    Prefetch the next queue_length items from iterator in a background thread.\n\n    Example:\n    >> for i in bg_iterator(range(10)):\n    >>     print(i)\n    \"\"\"\n\n    class _End:\n        pass\n\n    def __init__(self, generator: Iterable, maxsize: int) -> None:\n        threading.Thread.__init__(self)\n        self.queue: Queue = Queue(maxsize)\n        self.generator = generator\n        self.daemon = True\n        self.start()\n\n    def run(self) -> None:\n        for item in self.generator:\n            self.queue.put(item)\n        self.queue.put(self._End)\n\n    def __iter__(self) -> Any:\n        return self\n\n    def __next__(self) -> Any:\n        next_item = self.queue.get()\n        if next_item == self._End:\n            raise StopIteration\n        return next_item\n\n    # Required for Python 2.7 compatibility\n    def next(self) -> Any:\n        return self.__next__()\n\n\ndef bg_iterator(iterable: Iterable, maxsize: int) -> Any:\n    return _ThreadedIterator(iterable, maxsize=maxsize)\n", "path": "torchaudio/datasets/utils.py"}], "after_files": [{"content": "import csv\nimport errno\nimport hashlib\nimport logging\nimport os\nimport sys\nimport tarfile\nimport threading\nimport zipfile\nfrom _io import TextIOWrapper\nfrom queue import Queue\nfrom typing import Any, Iterable, List, Optional, Tuple, Union\n\nimport torch\nimport urllib\nimport urllib.request\nfrom torch.utils.data import Dataset\nfrom torch.utils.model_zoo import tqdm\n\n\ndef unicode_csv_reader(unicode_csv_data: TextIOWrapper, **kwargs: Any) -> Any:\n    r\"\"\"Since the standard csv library does not handle unicode in Python 2, we need a wrapper.\n    Borrowed and slightly modified from the Python docs:\n    https://docs.python.org/2/library/csv.html#csv-examples\n    Args:\n        unicode_csv_data (TextIOWrapper): unicode csv data (see example below)\n\n    Examples:\n        >>> from torchaudio.datasets.utils import unicode_csv_reader\n        >>> import io\n        >>> with io.open(data_path, encoding=\"utf8\") as f:\n        >>>     reader = unicode_csv_reader(f)\n    \"\"\"\n\n    # Fix field larger than field limit error\n    maxInt = sys.maxsize\n    while True:\n        # decrease the maxInt value by factor 10\n        # as long as the OverflowError occurs.\n        try:\n            csv.field_size_limit(maxInt)\n            break\n        except OverflowError:\n            maxInt = int(maxInt / 10)\n    csv.field_size_limit(maxInt)\n\n    for line in csv.reader(unicode_csv_data, **kwargs):\n        yield line\n\n\ndef makedir_exist_ok(dirpath: str) -> None:\n    \"\"\"\n    Python2 support for os.makedirs(.., exist_ok=True)\n    \"\"\"\n    try:\n        os.makedirs(dirpath)\n    except OSError as e:\n        if e.errno == errno.EEXIST:\n            pass\n        else:\n            raise\n\n\ndef stream_url(url: str,\n               start_byte: Optional[int] = None,\n               block_size: int = 32 * 1024,\n               progress_bar: bool = True) -> Iterable:\n    \"\"\"Stream url by chunk\n\n    Args:\n        url (str): Url.\n        start_byte (int, optional): Start streaming at that point (Default: ``None``).\n        block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n    \"\"\"\n\n    # If we already have the whole file, there is no need to download it again\n    req = urllib.request.Request(url, method=\"HEAD\")\n    url_size = int(urllib.request.urlopen(req).info().get(\"Content-Length\", -1))\n    if url_size == start_byte:\n        return\n\n    req = urllib.request.Request(url)\n    if start_byte:\n        req.headers[\"Range\"] = \"bytes={}-\".format(start_byte)\n\n    with urllib.request.urlopen(req) as upointer, tqdm(\n        unit=\"B\",\n        unit_scale=True,\n        unit_divisor=1024,\n        total=url_size,\n        disable=not progress_bar,\n    ) as pbar:\n\n        num_bytes = 0\n        while True:\n            chunk = upointer.read(block_size)\n            if not chunk:\n                break\n            yield chunk\n            num_bytes += len(chunk)\n            pbar.update(len(chunk))\n\n\ndef download_url(url: str,\n                 download_folder: str,\n                 filename: Optional[str] = None,\n                 hash_value: Optional[str] = None,\n                 hash_type: str = \"sha256\",\n                 progress_bar: bool = True,\n                 resume: bool = False) -> None:\n    \"\"\"Download file to disk.\n\n    Args:\n        url (str): Url.\n        download_folder (str): Folder to download file.\n        filename (str, optional): Name of downloaded file. If None, it is inferred from the url (Default: ``None``).\n        hash_value (str, optional): Hash for url (Default: ``None``).\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n        progress_bar (bool, optional): Display a progress bar (Default: ``True``).\n        resume (bool, optional): Enable resuming download (Default: ``False``).\n    \"\"\"\n\n    req = urllib.request.Request(url, method=\"HEAD\")\n    req_info = urllib.request.urlopen(req).info()\n\n    # Detect filename\n    filename = filename or req_info.get_filename() or os.path.basename(url)\n    filepath = os.path.join(download_folder, filename)\n    if resume and os.path.exists(filepath):\n        mode = \"ab\"\n        local_size: Optional[int] = os.path.getsize(filepath)\n\n    elif not resume and os.path.exists(filepath):\n        raise RuntimeError(\n            \"{} already exists. Delete the file manually and retry.\".format(filepath)\n        )\n    else:\n        mode = \"wb\"\n        local_size = None\n\n    if hash_value and local_size == int(req_info.get(\"Content-Length\", -1)):\n        with open(filepath, \"rb\") as file_obj:\n            if validate_file(file_obj, hash_value, hash_type):\n                return\n        raise RuntimeError(\n            \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                filepath\n            )\n        )\n\n    with open(filepath, mode) as fpointer:\n        for chunk in stream_url(url, start_byte=local_size, progress_bar=progress_bar):\n            fpointer.write(chunk)\n\n    with open(filepath, \"rb\") as file_obj:\n        if hash_value and not validate_file(file_obj, hash_value, hash_type):\n            raise RuntimeError(\n                \"The hash of {} does not match. Delete the file manually and retry.\".format(\n                    filepath\n                )\n            )\n\n\ndef validate_file(file_obj: Any, hash_value: str, hash_type: str = \"sha256\") -> bool:\n    \"\"\"Validate a given file object with its hash.\n\n    Args:\n        file_obj: File object to read from.\n        hash_value (str): Hash for url.\n        hash_type (str, optional): Hash type, among \"sha256\" and \"md5\" (Default: ``\"sha256\"``).\n\n    Returns:\n        bool: return True if its a valid file, else False.\n    \"\"\"\n\n    if hash_type == \"sha256\":\n        hash_func = hashlib.sha256()\n    elif hash_type == \"md5\":\n        hash_func = hashlib.md5()\n    else:\n        raise ValueError\n\n    while True:\n        # Read by chunk to avoid filling memory\n        chunk = file_obj.read(1024 ** 2)\n        if not chunk:\n            break\n        hash_func.update(chunk)\n\n    return hash_func.hexdigest() == hash_value\n\n\ndef extract_archive(from_path: str, to_path: Optional[str] = None, overwrite: bool = False) -> List[str]:\n    \"\"\"Extract archive.\n    Args:\n        from_path (str): the path of the archive.\n        to_path (str, optional): the root path of the extraced files (directory of from_path) (Default: ``None``)\n        overwrite (bool, optional): overwrite existing files (Default: ``False``)\n\n    Returns:\n        list: List of paths to extracted files even if not overwritten.\n\n    Examples:\n        >>> url = 'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz'\n        >>> from_path = './validation.tar.gz'\n        >>> to_path = './'\n        >>> torchaudio.datasets.utils.download_from_url(url, from_path)\n        >>> torchaudio.datasets.utils.extract_archive(from_path, to_path)\n    \"\"\"\n\n    if to_path is None:\n        to_path = os.path.dirname(from_path)\n\n    try:\n        with tarfile.open(from_path, \"r\") as tar:\n            logging.info(\"Opened tar file {}.\".format(from_path))\n            files = []\n            for file_ in tar:  # type: Any\n                file_path = os.path.join(to_path, file_.name)\n                if file_.isfile():\n                    files.append(file_path)\n                    if os.path.exists(file_path):\n                        logging.info(\"{} already extracted.\".format(file_path))\n                        if not overwrite:\n                            continue\n                tar.extract(file_, to_path)\n            return files\n    except tarfile.ReadError:\n        pass\n\n    try:\n        with zipfile.ZipFile(from_path, \"r\") as zfile:\n            logging.info(\"Opened zip file {}.\".format(from_path))\n            files = zfile.namelist()\n            for file_ in files:\n                file_path = os.path.join(to_path, file_)\n                if os.path.exists(file_path):\n                    logging.info(\"{} already extracted.\".format(file_path))\n                    if not overwrite:\n                        continue\n                zfile.extract(file_, to_path)\n        return files\n    except zipfile.BadZipFile:\n        pass\n\n    raise NotImplementedError(\"We currently only support tar.gz, tgz, and zip achives.\")\n\n\ndef walk_files(root: str,\n               suffix: Union[str, Tuple[str]],\n               prefix: bool = False,\n               remove_suffix: bool = False) -> Iterable[str]:\n    \"\"\"List recursively all files ending with a suffix at a given root\n    Args:\n        root (str): Path to directory whose folders need to be listed\n        suffix (str or tuple): Suffix of the files to match, e.g. '.png' or ('.jpg', '.png').\n            It uses the Python \"str.endswith\" method and is passed directly\n        prefix (bool, optional): If true, prepends the full path to each result, otherwise\n            only returns the name of the files found (Default: ``False``)\n        remove_suffix (bool, optional): If true, removes the suffix to each result defined in suffix,\n            otherwise will return the result as found (Default: ``False``).\n    \"\"\"\n\n    root = os.path.expanduser(root)\n\n    for dirpath, dirs, files in os.walk(root):\n        dirs.sort()\n        # `dirs` is the list used in os.walk function and by sorting it in-place here, we change the\n        # behavior of os.walk to traverse sub directory alphabetically\n        # see also\n        # https://stackoverflow.com/questions/6670029/can-i-force-python3s-os-walk-to-visit-directories-in-alphabetical-order-how#comment71993866_6670926\n        files.sort()\n        for f in files:\n            if f.endswith(suffix):\n\n                if remove_suffix:\n                    f = f[: -len(suffix)]\n\n                if prefix:\n                    f = os.path.join(dirpath, f)\n\n                yield f\n\n\nclass _DiskCache(Dataset):\n    \"\"\"\n    Wrap a dataset so that, whenever a new item is returned, it is saved to disk.\n    \"\"\"\n\n    def __init__(self, dataset: Dataset, location: str = \".cached\") -> None:\n        self.dataset = dataset\n        self.location = location\n\n        self._id = id(self)\n        self._cache: List = [None] * len(dataset)\n\n    def __getitem__(self, n: int) -> Any:\n        if self._cache[n]:\n            f = self._cache[n]\n            return torch.load(f)\n\n        f = str(self._id) + \"-\" + str(n)\n        f = os.path.join(self.location, f)\n        item = self.dataset[n]\n\n        self._cache[n] = f\n        makedir_exist_ok(self.location)\n        torch.save(item, f)\n\n        return item\n\n    def __len__(self) -> int:\n        return len(self.dataset)\n\n\ndef diskcache_iterator(dataset: Dataset, location: str = \".cached\") -> Dataset:\n    return _DiskCache(dataset, location)\n\n\nclass _ThreadedIterator(threading.Thread):\n    \"\"\"\n    Prefetch the next queue_length items from iterator in a background thread.\n\n    Example:\n    >> for i in bg_iterator(range(10)):\n    >>     print(i)\n    \"\"\"\n\n    class _End:\n        pass\n\n    def __init__(self, generator: Iterable, maxsize: int) -> None:\n        threading.Thread.__init__(self)\n        self.queue: Queue = Queue(maxsize)\n        self.generator = generator\n        self.daemon = True\n        self.start()\n\n    def run(self) -> None:\n        for item in self.generator:\n            self.queue.put(item)\n        self.queue.put(self._End)\n\n    def __iter__(self) -> Any:\n        return self\n\n    def __next__(self) -> Any:\n        next_item = self.queue.get()\n        if next_item == self._End:\n            raise StopIteration\n        return next_item\n\n    # Required for Python 2.7 compatibility\n    def next(self) -> Any:\n        return self.__next__()\n\n\ndef bg_iterator(iterable: Iterable, maxsize: int) -> Any:\n    return _ThreadedIterator(iterable, maxsize=maxsize)\n", "path": "torchaudio/datasets/utils.py"}]}