We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
DS2 reader: Move local data variable from global into class DataGenerator.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `deep_speech_2/data_utils/data.py`
Content:
```
1 """Contains data generator for orgnaizing various audio data preprocessing
2 pipeline and offering data reader interface of PaddlePaddle requirements.
3 """
4 from __future__ import absolute_import
5 from __future__ import division
6 from __future__ import print_function
7 
8 import random
9 import tarfile
10 import multiprocessing
11 import numpy as np
12 import paddle.v2 as paddle
13 from threading import local
14 from data_utils import utils
15 from data_utils.augmentor.augmentation import AugmentationPipeline
16 from data_utils.featurizer.speech_featurizer import SpeechFeaturizer
17 from data_utils.speech import SpeechSegment
18 from data_utils.normalizer import FeatureNormalizer
19 
20 # for caching tar files info
21 local_data = local()
22 local_data.tar2info = {}
23 local_data.tar2object = {}
24 
25 
26 class DataGenerator(object):
27     """
28     DataGenerator provides basic audio data preprocessing pipeline, and offers
29     data reader interfaces of PaddlePaddle requirements.
30 
31     :param vocab_filepath: Vocabulary filepath for indexing tokenized
32                            transcripts.
33     :type vocab_filepath: basestring
34     :param mean_std_filepath: File containing the pre-computed mean and stddev.
35     :type mean_std_filepath: None|basestring
36     :param augmentation_config: Augmentation configuration in json string.
37                                 Details see AugmentationPipeline.__doc__.
38     :type augmentation_config: str
39     :param max_duration: Audio with duration (in seconds) greater than
40                          this will be discarded.
41     :type max_duration: float
42     :param min_duration: Audio with duration (in seconds) smaller than
43                          this will be discarded.
44     :type min_duration: float
45     :param stride_ms: Striding size (in milliseconds) for generating frames.
46     :type stride_ms: float
47     :param window_ms: Window size (in milliseconds) for generating frames.
48     :type window_ms: float
49     :param max_freq: Used when specgram_type is 'linear', only FFT bins
50                      corresponding to frequencies between [0, max_freq] are
51                      returned.
52     :types max_freq: None|float
53     :param specgram_type: Specgram feature type. Options: 'linear'.
54     :type specgram_type: str
55     :param use_dB_normalization: Whether to normalize the audio to -20 dB
56                                 before extracting the features.
57     :type use_dB_normalization: bool
58     :param num_threads: Number of CPU threads for processing data.
59     :type num_threads: int
60     :param random_seed: Random seed.
61     :type random_seed: int
62     """
63 
64     def __init__(self,
65                  vocab_filepath,
66                  mean_std_filepath,
67                  augmentation_config='{}',
68                  max_duration=float('inf'),
69                  min_duration=0.0,
70                  stride_ms=10.0,
71                  window_ms=20.0,
72                  max_freq=None,
73                  specgram_type='linear',
74                  use_dB_normalization=True,
75                  num_threads=multiprocessing.cpu_count() // 2,
76                  random_seed=0):
77         self._max_duration = max_duration
78         self._min_duration = min_duration
79         self._normalizer = FeatureNormalizer(mean_std_filepath)
80         self._augmentation_pipeline = AugmentationPipeline(
81             augmentation_config=augmentation_config, random_seed=random_seed)
82         self._speech_featurizer = SpeechFeaturizer(
83             vocab_filepath=vocab_filepath,
84             specgram_type=specgram_type,
85             stride_ms=stride_ms,
86             window_ms=window_ms,
87             max_freq=max_freq,
88             use_dB_normalization=use_dB_normalization)
89         self._num_threads = num_threads
90         self._rng = random.Random(random_seed)
91         self._epoch = 0
92 
93     def process_utterance(self, filename, transcript):
94         """Load, augment, featurize and normalize for speech data.
95 
96         :param filename: Audio filepath
97         :type filename: basestring | file
98         :param transcript: Transcription text.
99         :type transcript: basestring
100         :return: Tuple of audio feature tensor and list of token ids for
101                  transcription.
102         :rtype: tuple of (2darray, list)
103         """
104         speech_segment = SpeechSegment.from_file(filename, transcript)
105         self._augmentation_pipeline.transform_audio(speech_segment)
106         specgram, text_ids = self._speech_featurizer.featurize(speech_segment)
107         specgram = self._normalizer.apply(specgram)
108         return specgram, text_ids
109 
110     def batch_reader_creator(self,
111                              manifest_path,
112                              batch_size,
113                              min_batch_size=1,
114                              padding_to=-1,
115                              flatten=False,
116                              sortagrad=False,
117                              shuffle_method="batch_shuffle"):
118         """
119         Batch data reader creator for audio data. Return a callable generator
120         function to produce batches of data.
121 
122         Audio features within one batch will be padded with zeros to have the
123         same shape, or a user-defined shape.
124 
125         :param manifest_path: Filepath of manifest for audio files.
126         :type manifest_path: basestring
127         :param batch_size: Number of instances in a batch.
128         :type batch_size: int
129         :param min_batch_size: Any batch with batch size smaller than this will
130                                be discarded. (To be deprecated in the future.)
131         :type min_batch_size: int
132         :param padding_to:  If set -1, the maximun shape in the batch
133                             will be used as the target shape for padding.
134                             Otherwise, `padding_to` will be the target shape.
135         :type padding_to: int
136         :param flatten: If set True, audio features will be flatten to 1darray.
137         :type flatten: bool
138         :param sortagrad: If set True, sort the instances by audio duration
139                           in the first epoch for speed up training.
140         :type sortagrad: bool
141         :param shuffle_method: Shuffle method. Options:
142                                 '' or None: no shuffle.
143                                 'instance_shuffle': instance-wise shuffle.
144                                 'batch_shuffle': similarly-sized instances are
145                                                  put into batches, and then
146                                                  batch-wise shuffle the batches.
147                                                  For more details, please see
148                                                  ``_batch_shuffle.__doc__``.
149                                 'batch_shuffle_clipped': 'batch_shuffle' with
150                                                          head shift and tail
151                                                          clipping. For more
152                                                          details, please see
153                                                          ``_batch_shuffle``.
154                               If sortagrad is True, shuffle is disabled
155                               for the first epoch.
156         :type shuffle_method: None|str
157         :return: Batch reader function, producing batches of data when called.
158         :rtype: callable
159         """
160 
161         def batch_reader():
162             # read manifest
163             manifest = utils.read_manifest(
164                 manifest_path=manifest_path,
165                 max_duration=self._max_duration,
166                 min_duration=self._min_duration)
167             # sort (by duration) or batch-wise shuffle the manifest
168             if self._epoch == 0 and sortagrad:
169                 manifest.sort(key=lambda x: x["duration"])
170             else:
171                 if shuffle_method == "batch_shuffle":
172                     manifest = self._batch_shuffle(
173                         manifest, batch_size, clipped=False)
174                 elif shuffle_method == "batch_shuffle_clipped":
175                     manifest = self._batch_shuffle(
176                         manifest, batch_size, clipped=True)
177                 elif shuffle_method == "instance_shuffle":
178                     self._rng.shuffle(manifest)
179                 elif shuffle_method == None:
180                     pass
181                 else:
182                     raise ValueError("Unknown shuffle method %s." %
183                                      shuffle_method)
184             # prepare batches
185             instance_reader = self._instance_reader_creator(manifest)
186             batch = []
187             for instance in instance_reader():
188                 batch.append(instance)
189                 if len(batch) == batch_size:
190                     yield self._padding_batch(batch, padding_to, flatten)
191                     batch = []
192             if len(batch) >= min_batch_size:
193                 yield self._padding_batch(batch, padding_to, flatten)
194             self._epoch += 1
195 
196         return batch_reader
197 
198     @property
199     def feeding(self):
200         """Returns data reader's feeding dict.
201 
202         :return: Data feeding dict.
203         :rtype: dict
204         """
205         return {"audio_spectrogram": 0, "transcript_text": 1}
206 
207     @property
208     def vocab_size(self):
209         """Return the vocabulary size.
210 
211         :return: Vocabulary size.
212         :rtype: int
213         """
214         return self._speech_featurizer.vocab_size
215 
216     @property
217     def vocab_list(self):
218         """Return the vocabulary in list.
219 
220         :return: Vocabulary in list.
221         :rtype: list
222         """
223         return self._speech_featurizer.vocab_list
224 
225     def _parse_tar(self, file):
226         """Parse a tar file to get a tarfile object
227         and a map containing tarinfoes
228         """
229         result = {}
230         f = tarfile.open(file)
231         for tarinfo in f.getmembers():
232             result[tarinfo.name] = tarinfo
233         return f, result
234 
235     def _get_file_object(self, file):
236         """Get file object by file path.
237 
238         If file startwith tar, it will return a tar file object
239         and cached tar file info for next reading request.
240         It will return file directly, if the type of file is not str.
241         """
242         if file.startswith('tar:'):
243             tarpath, filename = file.split(':', 1)[1].split('#', 1)
244             if 'tar2info' not in local_data.__dict__:
245                 local_data.tar2info = {}
246             if 'tar2object' not in local_data.__dict__:
247                 local_data.tar2object = {}
248             if tarpath not in local_data.tar2info:
249                 object, infoes = self._parse_tar(tarpath)
250                 local_data.tar2info[tarpath] = infoes
251                 local_data.tar2object[tarpath] = object
252             return local_data.tar2object[tarpath].extractfile(
253                 local_data.tar2info[tarpath][filename])
254         else:
255             return open(file, 'r')
256 
257     def _instance_reader_creator(self, manifest):
258         """
259         Instance reader creator. Create a callable function to produce
260         instances of data.
261 
262         Instance: a tuple of ndarray of audio spectrogram and a list of
263         token indices for transcript.
264         """
265 
266         def reader():
267             for instance in manifest:
268                 yield instance
269 
270         def mapper(instance):
271             return self.process_utterance(
272                 self._get_file_object(instance["audio_filepath"]),
273                 instance["text"])
274 
275         return paddle.reader.xmap_readers(
276             mapper, reader, self._num_threads, 1024, order=True)
277 
278     def _padding_batch(self, batch, padding_to=-1, flatten=False):
279         """
280         Padding audio features with zeros to make them have the same shape (or
281         a user-defined shape) within one bach.
282 
283         If ``padding_to`` is -1, the maximun shape in the batch will be used
284         as the target shape for padding. Otherwise, `padding_to` will be the
285         target shape (only refers to the second axis).
286 
287         If `flatten` is True, features will be flatten to 1darray.
288         """
289         new_batch = []
290         # get target shape
291         max_length = max([audio.shape[1] for audio, text in batch])
292         if padding_to != -1:
293             if padding_to < max_length:
294                 raise ValueError("If padding_to is not -1, it should be larger "
295                                  "than any instance's shape in the batch")
296             max_length = padding_to
297         # padding
298         for audio, text in batch:
299             padded_audio = np.zeros([audio.shape[0], max_length])
300             padded_audio[:, :audio.shape[1]] = audio
301             if flatten:
302                 padded_audio = padded_audio.flatten()
303             new_batch.append((padded_audio, text))
304         return new_batch
305 
306     def _batch_shuffle(self, manifest, batch_size, clipped=False):
307         """Put similarly-sized instances into minibatches for better efficiency
308         and make a batch-wise shuffle.
309 
310         1. Sort the audio clips by duration.
311         2. Generate a random number `k`, k in [0, batch_size).
312         3. Randomly shift `k` instances in order to create different batches
313            for different epochs. Create minibatches.
314         4. Shuffle the minibatches.
315 
316         :param manifest: Manifest contents. List of dict.
317         :type manifest: list
318         :param batch_size: Batch size. This size is also used for generate
319                            a random number for batch shuffle.
320         :type batch_size: int
321         :param clipped: Whether to clip the heading (small shift) and trailing
322                         (incomplete batch) instances.
323         :type clipped: bool
324         :return: Batch shuffled mainifest.
325         :rtype: list
326         """
327         manifest.sort(key=lambda x: x["duration"])
328         shift_len = self._rng.randint(0, batch_size - 1)
329         batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)
330         self._rng.shuffle(batch_manifest)
331         batch_manifest = list(sum(batch_manifest, ()))
332         if not clipped:
333             res_len = len(manifest) - shift_len - len(batch_manifest)
334             batch_manifest.extend(manifest[-res_len:])
335             batch_manifest.extend(manifest[0:shift_len])
336         return batch_manifest
337
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/deep_speech_2/data_utils/data.py b/deep_speech_2/data_utils/data.py
--- a/deep_speech_2/data_utils/data.py
+++ b/deep_speech_2/data_utils/data.py
@@ -17,11 +17,6 @@
 from data_utils.speech import SpeechSegment
 from data_utils.normalizer import FeatureNormalizer
 
-# for caching tar files info
-local_data = local()
-local_data.tar2info = {}
-local_data.tar2object = {}
-
 
 class DataGenerator(object):
     """
@@ -89,6 +84,10 @@
         self._num_threads = num_threads
         self._rng = random.Random(random_seed)
         self._epoch = 0
+        # for caching tar files info
+        self.local_data = local()
+        self.local_data.tar2info = {}
+        self.local_data.tar2object = {}
 
     def process_utterance(self, filename, transcript):
         """Load, augment, featurize and normalize for speech data.
@@ -241,16 +240,16 @@
         """
         if file.startswith('tar:'):
             tarpath, filename = file.split(':', 1)[1].split('#', 1)
-            if 'tar2info' not in local_data.__dict__:
-                local_data.tar2info = {}
-            if 'tar2object' not in local_data.__dict__:
-                local_data.tar2object = {}
-            if tarpath not in local_data.tar2info:
+            if 'tar2info' not in self.local_data.__dict__:
+                self.local_data.tar2info = {}
+            if 'tar2object' not in self.local_data.__dict__:
+                self.local_data.tar2object = {}
+            if tarpath not in self.local_data.tar2info:
                 object, infoes = self._parse_tar(tarpath)
-                local_data.tar2info[tarpath] = infoes
-                local_data.tar2object[tarpath] = object
-            return local_data.tar2object[tarpath].extractfile(
-                local_data.tar2info[tarpath][filename])
+                self.local_data.tar2info[tarpath] = infoes
+                self.local_data.tar2object[tarpath] = object
+            return self.local_data.tar2object[tarpath].extractfile(
+                self.local_data.tar2info[tarpath][filename])
         else:
             return open(file, 'r')

{"golden_diff": "diff --git a/deep_speech_2/data_utils/data.py b/deep_speech_2/data_utils/data.py\n--- a/deep_speech_2/data_utils/data.py\n+++ b/deep_speech_2/data_utils/data.py\n@@ -17,11 +17,6 @@\n from data_utils.speech import SpeechSegment\n from data_utils.normalizer import FeatureNormalizer\n \n-# for caching tar files info\n-local_data = local()\n-local_data.tar2info = {}\n-local_data.tar2object = {}\n-\n \n class DataGenerator(object):\n     \"\"\"\n@@ -89,6 +84,10 @@\n         self._num_threads = num_threads\n         self._rng = random.Random(random_seed)\n         self._epoch = 0\n+        # for caching tar files info\n+        self.local_data = local()\n+        self.local_data.tar2info = {}\n+        self.local_data.tar2object = {}\n \n     def process_utterance(self, filename, transcript):\n         \"\"\"Load, augment, featurize and normalize for speech data.\n@@ -241,16 +240,16 @@\n         \"\"\"\n         if file.startswith('tar:'):\n             tarpath, filename = file.split(':', 1)[1].split('#', 1)\n-            if 'tar2info' not in local_data.__dict__:\n-                local_data.tar2info = {}\n-            if 'tar2object' not in local_data.__dict__:\n-                local_data.tar2object = {}\n-            if tarpath not in local_data.tar2info:\n+            if 'tar2info' not in self.local_data.__dict__:\n+                self.local_data.tar2info = {}\n+            if 'tar2object' not in self.local_data.__dict__:\n+                self.local_data.tar2object = {}\n+            if tarpath not in self.local_data.tar2info:\n                 object, infoes = self._parse_tar(tarpath)\n-                local_data.tar2info[tarpath] = infoes\n-                local_data.tar2object[tarpath] = object\n-            return local_data.tar2object[tarpath].extractfile(\n-                local_data.tar2info[tarpath][filename])\n+                self.local_data.tar2info[tarpath] = infoes\n+                self.local_data.tar2object[tarpath] = object\n+            return self.local_data.tar2object[tarpath].extractfile(\n+                self.local_data.tar2info[tarpath][filename])\n         else:\n             return open(file, 'r')\n", "issue": "DS2 reader: Move local data variable from global into class DataGenerator.\n\n", "before_files": [{"content": "\"\"\"Contains data generator for orgnaizing various audio data preprocessing\npipeline and offering data reader interface of PaddlePaddle requirements.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport random\nimport tarfile\nimport multiprocessing\nimport numpy as np\nimport paddle.v2 as paddle\nfrom threading import local\nfrom data_utils import utils\nfrom data_utils.augmentor.augmentation import AugmentationPipeline\nfrom data_utils.featurizer.speech_featurizer import SpeechFeaturizer\nfrom data_utils.speech import SpeechSegment\nfrom data_utils.normalizer import FeatureNormalizer\n\n# for caching tar files info\nlocal_data = local()\nlocal_data.tar2info = {}\nlocal_data.tar2object = {}\n\n\nclass DataGenerator(object):\n    \"\"\"\n    DataGenerator provides basic audio data preprocessing pipeline, and offers\n    data reader interfaces of PaddlePaddle requirements.\n\n    :param vocab_filepath: Vocabulary filepath for indexing tokenized\n                           transcripts.\n    :type vocab_filepath: basestring\n    :param mean_std_filepath: File containing the pre-computed mean and stddev.\n    :type mean_std_filepath: None|basestring\n    :param augmentation_config: Augmentation configuration in json string.\n                                Details see AugmentationPipeline.__doc__.\n    :type augmentation_config: str\n    :param max_duration: Audio with duration (in seconds) greater than\n                         this will be discarded.\n    :type max_duration: float\n    :param min_duration: Audio with duration (in seconds) smaller than\n                         this will be discarded.\n    :type min_duration: float\n    :param stride_ms: Striding size (in milliseconds) for generating frames.\n    :type stride_ms: float\n    :param window_ms: Window size (in milliseconds) for generating frames.\n    :type window_ms: float\n    :param max_freq: Used when specgram_type is 'linear', only FFT bins\n                     corresponding to frequencies between [0, max_freq] are\n                     returned.\n    :types max_freq: None|float\n    :param specgram_type: Specgram feature type. Options: 'linear'.\n    :type specgram_type: str\n    :param use_dB_normalization: Whether to normalize the audio to -20 dB\n                                before extracting the features.\n    :type use_dB_normalization: bool\n    :param num_threads: Number of CPU threads for processing data.\n    :type num_threads: int\n    :param random_seed: Random seed.\n    :type random_seed: int\n    \"\"\"\n\n    def __init__(self,\n                 vocab_filepath,\n                 mean_std_filepath,\n                 augmentation_config='{}',\n                 max_duration=float('inf'),\n                 min_duration=0.0,\n                 stride_ms=10.0,\n                 window_ms=20.0,\n                 max_freq=None,\n                 specgram_type='linear',\n                 use_dB_normalization=True,\n                 num_threads=multiprocessing.cpu_count() // 2,\n                 random_seed=0):\n        self._max_duration = max_duration\n        self._min_duration = min_duration\n        self._normalizer = FeatureNormalizer(mean_std_filepath)\n        self._augmentation_pipeline = AugmentationPipeline(\n            augmentation_config=augmentation_config, random_seed=random_seed)\n        self._speech_featurizer = SpeechFeaturizer(\n            vocab_filepath=vocab_filepath,\n            specgram_type=specgram_type,\n            stride_ms=stride_ms,\n            window_ms=window_ms,\n            max_freq=max_freq,\n            use_dB_normalization=use_dB_normalization)\n        self._num_threads = num_threads\n        self._rng = random.Random(random_seed)\n        self._epoch = 0\n\n    def process_utterance(self, filename, transcript):\n        \"\"\"Load, augment, featurize and normalize for speech data.\n\n        :param filename: Audio filepath\n        :type filename: basestring | file\n        :param transcript: Transcription text.\n        :type transcript: basestring\n        :return: Tuple of audio feature tensor and list of token ids for\n                 transcription.\n        :rtype: tuple of (2darray, list)\n        \"\"\"\n        speech_segment = SpeechSegment.from_file(filename, transcript)\n        self._augmentation_pipeline.transform_audio(speech_segment)\n        specgram, text_ids = self._speech_featurizer.featurize(speech_segment)\n        specgram = self._normalizer.apply(specgram)\n        return specgram, text_ids\n\n    def batch_reader_creator(self,\n                             manifest_path,\n                             batch_size,\n                             min_batch_size=1,\n                             padding_to=-1,\n                             flatten=False,\n                             sortagrad=False,\n                             shuffle_method=\"batch_shuffle\"):\n        \"\"\"\n        Batch data reader creator for audio data. Return a callable generator\n        function to produce batches of data.\n\n        Audio features within one batch will be padded with zeros to have the\n        same shape, or a user-defined shape.\n\n        :param manifest_path: Filepath of manifest for audio files.\n        :type manifest_path: basestring\n        :param batch_size: Number of instances in a batch.\n        :type batch_size: int\n        :param min_batch_size: Any batch with batch size smaller than this will\n                               be discarded. (To be deprecated in the future.)\n        :type min_batch_size: int\n        :param padding_to:  If set -1, the maximun shape in the batch\n                            will be used as the target shape for padding.\n                            Otherwise, `padding_to` will be the target shape.\n        :type padding_to: int\n        :param flatten: If set True, audio features will be flatten to 1darray.\n        :type flatten: bool\n        :param sortagrad: If set True, sort the instances by audio duration\n                          in the first epoch for speed up training.\n        :type sortagrad: bool\n        :param shuffle_method: Shuffle method. Options:\n                                '' or None: no shuffle.\n                                'instance_shuffle': instance-wise shuffle.\n                                'batch_shuffle': similarly-sized instances are\n                                                 put into batches, and then\n                                                 batch-wise shuffle the batches.\n                                                 For more details, please see\n                                                 ``_batch_shuffle.__doc__``.\n                                'batch_shuffle_clipped': 'batch_shuffle' with\n                                                         head shift and tail\n                                                         clipping. For more\n                                                         details, please see\n                                                         ``_batch_shuffle``.\n                              If sortagrad is True, shuffle is disabled\n                              for the first epoch.\n        :type shuffle_method: None|str\n        :return: Batch reader function, producing batches of data when called.\n        :rtype: callable\n        \"\"\"\n\n        def batch_reader():\n            # read manifest\n            manifest = utils.read_manifest(\n                manifest_path=manifest_path,\n                max_duration=self._max_duration,\n                min_duration=self._min_duration)\n            # sort (by duration) or batch-wise shuffle the manifest\n            if self._epoch == 0 and sortagrad:\n                manifest.sort(key=lambda x: x[\"duration\"])\n            else:\n                if shuffle_method == \"batch_shuffle\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=False)\n                elif shuffle_method == \"batch_shuffle_clipped\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=True)\n                elif shuffle_method == \"instance_shuffle\":\n                    self._rng.shuffle(manifest)\n                elif shuffle_method == None:\n                    pass\n                else:\n                    raise ValueError(\"Unknown shuffle method %s.\" %\n                                     shuffle_method)\n            # prepare batches\n            instance_reader = self._instance_reader_creator(manifest)\n            batch = []\n            for instance in instance_reader():\n                batch.append(instance)\n                if len(batch) == batch_size:\n                    yield self._padding_batch(batch, padding_to, flatten)\n                    batch = []\n            if len(batch) >= min_batch_size:\n                yield self._padding_batch(batch, padding_to, flatten)\n            self._epoch += 1\n\n        return batch_reader\n\n    @property\n    def feeding(self):\n        \"\"\"Returns data reader's feeding dict.\n\n        :return: Data feeding dict.\n        :rtype: dict\n        \"\"\"\n        return {\"audio_spectrogram\": 0, \"transcript_text\": 1}\n\n    @property\n    def vocab_size(self):\n        \"\"\"Return the vocabulary size.\n\n        :return: Vocabulary size.\n        :rtype: int\n        \"\"\"\n        return self._speech_featurizer.vocab_size\n\n    @property\n    def vocab_list(self):\n        \"\"\"Return the vocabulary in list.\n\n        :return: Vocabulary in list.\n        :rtype: list\n        \"\"\"\n        return self._speech_featurizer.vocab_list\n\n    def _parse_tar(self, file):\n        \"\"\"Parse a tar file to get a tarfile object\n        and a map containing tarinfoes\n        \"\"\"\n        result = {}\n        f = tarfile.open(file)\n        for tarinfo in f.getmembers():\n            result[tarinfo.name] = tarinfo\n        return f, result\n\n    def _get_file_object(self, file):\n        \"\"\"Get file object by file path.\n\n        If file startwith tar, it will return a tar file object\n        and cached tar file info for next reading request.\n        It will return file directly, if the type of file is not str.\n        \"\"\"\n        if file.startswith('tar:'):\n            tarpath, filename = file.split(':', 1)[1].split('#', 1)\n            if 'tar2info' not in local_data.__dict__:\n                local_data.tar2info = {}\n            if 'tar2object' not in local_data.__dict__:\n                local_data.tar2object = {}\n            if tarpath not in local_data.tar2info:\n                object, infoes = self._parse_tar(tarpath)\n                local_data.tar2info[tarpath] = infoes\n                local_data.tar2object[tarpath] = object\n            return local_data.tar2object[tarpath].extractfile(\n                local_data.tar2info[tarpath][filename])\n        else:\n            return open(file, 'r')\n\n    def _instance_reader_creator(self, manifest):\n        \"\"\"\n        Instance reader creator. Create a callable function to produce\n        instances of data.\n\n        Instance: a tuple of ndarray of audio spectrogram and a list of\n        token indices for transcript.\n        \"\"\"\n\n        def reader():\n            for instance in manifest:\n                yield instance\n\n        def mapper(instance):\n            return self.process_utterance(\n                self._get_file_object(instance[\"audio_filepath\"]),\n                instance[\"text\"])\n\n        return paddle.reader.xmap_readers(\n            mapper, reader, self._num_threads, 1024, order=True)\n\n    def _padding_batch(self, batch, padding_to=-1, flatten=False):\n        \"\"\"\n        Padding audio features with zeros to make them have the same shape (or\n        a user-defined shape) within one bach.\n\n        If ``padding_to`` is -1, the maximun shape in the batch will be used\n        as the target shape for padding. Otherwise, `padding_to` will be the\n        target shape (only refers to the second axis).\n\n        If `flatten` is True, features will be flatten to 1darray.\n        \"\"\"\n        new_batch = []\n        # get target shape\n        max_length = max([audio.shape[1] for audio, text in batch])\n        if padding_to != -1:\n            if padding_to < max_length:\n                raise ValueError(\"If padding_to is not -1, it should be larger \"\n                                 \"than any instance's shape in the batch\")\n            max_length = padding_to\n        # padding\n        for audio, text in batch:\n            padded_audio = np.zeros([audio.shape[0], max_length])\n            padded_audio[:, :audio.shape[1]] = audio\n            if flatten:\n                padded_audio = padded_audio.flatten()\n            new_batch.append((padded_audio, text))\n        return new_batch\n\n    def _batch_shuffle(self, manifest, batch_size, clipped=False):\n        \"\"\"Put similarly-sized instances into minibatches for better efficiency\n        and make a batch-wise shuffle.\n\n        1. Sort the audio clips by duration.\n        2. Generate a random number `k`, k in [0, batch_size).\n        3. Randomly shift `k` instances in order to create different batches\n           for different epochs. Create minibatches.\n        4. Shuffle the minibatches.\n\n        :param manifest: Manifest contents. List of dict.\n        :type manifest: list\n        :param batch_size: Batch size. This size is also used for generate\n                           a random number for batch shuffle.\n        :type batch_size: int\n        :param clipped: Whether to clip the heading (small shift) and trailing\n                        (incomplete batch) instances.\n        :type clipped: bool\n        :return: Batch shuffled mainifest.\n        :rtype: list\n        \"\"\"\n        manifest.sort(key=lambda x: x[\"duration\"])\n        shift_len = self._rng.randint(0, batch_size - 1)\n        batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)\n        self._rng.shuffle(batch_manifest)\n        batch_manifest = list(sum(batch_manifest, ()))\n        if not clipped:\n            res_len = len(manifest) - shift_len - len(batch_manifest)\n            batch_manifest.extend(manifest[-res_len:])\n            batch_manifest.extend(manifest[0:shift_len])\n        return batch_manifest\n", "path": "deep_speech_2/data_utils/data.py"}], "after_files": [{"content": "\"\"\"Contains data generator for orgnaizing various audio data preprocessing\npipeline and offering data reader interface of PaddlePaddle requirements.\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport random\nimport tarfile\nimport multiprocessing\nimport numpy as np\nimport paddle.v2 as paddle\nfrom threading import local\nfrom data_utils import utils\nfrom data_utils.augmentor.augmentation import AugmentationPipeline\nfrom data_utils.featurizer.speech_featurizer import SpeechFeaturizer\nfrom data_utils.speech import SpeechSegment\nfrom data_utils.normalizer import FeatureNormalizer\n\n\nclass DataGenerator(object):\n    \"\"\"\n    DataGenerator provides basic audio data preprocessing pipeline, and offers\n    data reader interfaces of PaddlePaddle requirements.\n\n    :param vocab_filepath: Vocabulary filepath for indexing tokenized\n                           transcripts.\n    :type vocab_filepath: basestring\n    :param mean_std_filepath: File containing the pre-computed mean and stddev.\n    :type mean_std_filepath: None|basestring\n    :param augmentation_config: Augmentation configuration in json string.\n                                Details see AugmentationPipeline.__doc__.\n    :type augmentation_config: str\n    :param max_duration: Audio with duration (in seconds) greater than\n                         this will be discarded.\n    :type max_duration: float\n    :param min_duration: Audio with duration (in seconds) smaller than\n                         this will be discarded.\n    :type min_duration: float\n    :param stride_ms: Striding size (in milliseconds) for generating frames.\n    :type stride_ms: float\n    :param window_ms: Window size (in milliseconds) for generating frames.\n    :type window_ms: float\n    :param max_freq: Used when specgram_type is 'linear', only FFT bins\n                     corresponding to frequencies between [0, max_freq] are\n                     returned.\n    :types max_freq: None|float\n    :param specgram_type: Specgram feature type. Options: 'linear'.\n    :type specgram_type: str\n    :param use_dB_normalization: Whether to normalize the audio to -20 dB\n                                before extracting the features.\n    :type use_dB_normalization: bool\n    :param num_threads: Number of CPU threads for processing data.\n    :type num_threads: int\n    :param random_seed: Random seed.\n    :type random_seed: int\n    \"\"\"\n\n    def __init__(self,\n                 vocab_filepath,\n                 mean_std_filepath,\n                 augmentation_config='{}',\n                 max_duration=float('inf'),\n                 min_duration=0.0,\n                 stride_ms=10.0,\n                 window_ms=20.0,\n                 max_freq=None,\n                 specgram_type='linear',\n                 use_dB_normalization=True,\n                 num_threads=multiprocessing.cpu_count() // 2,\n                 random_seed=0):\n        self._max_duration = max_duration\n        self._min_duration = min_duration\n        self._normalizer = FeatureNormalizer(mean_std_filepath)\n        self._augmentation_pipeline = AugmentationPipeline(\n            augmentation_config=augmentation_config, random_seed=random_seed)\n        self._speech_featurizer = SpeechFeaturizer(\n            vocab_filepath=vocab_filepath,\n            specgram_type=specgram_type,\n            stride_ms=stride_ms,\n            window_ms=window_ms,\n            max_freq=max_freq,\n            use_dB_normalization=use_dB_normalization)\n        self._num_threads = num_threads\n        self._rng = random.Random(random_seed)\n        self._epoch = 0\n        # for caching tar files info\n        self.local_data = local()\n        self.local_data.tar2info = {}\n        self.local_data.tar2object = {}\n\n    def process_utterance(self, filename, transcript):\n        \"\"\"Load, augment, featurize and normalize for speech data.\n\n        :param filename: Audio filepath\n        :type filename: basestring | file\n        :param transcript: Transcription text.\n        :type transcript: basestring\n        :return: Tuple of audio feature tensor and list of token ids for\n                 transcription.\n        :rtype: tuple of (2darray, list)\n        \"\"\"\n        speech_segment = SpeechSegment.from_file(filename, transcript)\n        self._augmentation_pipeline.transform_audio(speech_segment)\n        specgram, text_ids = self._speech_featurizer.featurize(speech_segment)\n        specgram = self._normalizer.apply(specgram)\n        return specgram, text_ids\n\n    def batch_reader_creator(self,\n                             manifest_path,\n                             batch_size,\n                             min_batch_size=1,\n                             padding_to=-1,\n                             flatten=False,\n                             sortagrad=False,\n                             shuffle_method=\"batch_shuffle\"):\n        \"\"\"\n        Batch data reader creator for audio data. Return a callable generator\n        function to produce batches of data.\n\n        Audio features within one batch will be padded with zeros to have the\n        same shape, or a user-defined shape.\n\n        :param manifest_path: Filepath of manifest for audio files.\n        :type manifest_path: basestring\n        :param batch_size: Number of instances in a batch.\n        :type batch_size: int\n        :param min_batch_size: Any batch with batch size smaller than this will\n                               be discarded. (To be deprecated in the future.)\n        :type min_batch_size: int\n        :param padding_to:  If set -1, the maximun shape in the batch\n                            will be used as the target shape for padding.\n                            Otherwise, `padding_to` will be the target shape.\n        :type padding_to: int\n        :param flatten: If set True, audio features will be flatten to 1darray.\n        :type flatten: bool\n        :param sortagrad: If set True, sort the instances by audio duration\n                          in the first epoch for speed up training.\n        :type sortagrad: bool\n        :param shuffle_method: Shuffle method. Options:\n                                '' or None: no shuffle.\n                                'instance_shuffle': instance-wise shuffle.\n                                'batch_shuffle': similarly-sized instances are\n                                                 put into batches, and then\n                                                 batch-wise shuffle the batches.\n                                                 For more details, please see\n                                                 ``_batch_shuffle.__doc__``.\n                                'batch_shuffle_clipped': 'batch_shuffle' with\n                                                         head shift and tail\n                                                         clipping. For more\n                                                         details, please see\n                                                         ``_batch_shuffle``.\n                              If sortagrad is True, shuffle is disabled\n                              for the first epoch.\n        :type shuffle_method: None|str\n        :return: Batch reader function, producing batches of data when called.\n        :rtype: callable\n        \"\"\"\n\n        def batch_reader():\n            # read manifest\n            manifest = utils.read_manifest(\n                manifest_path=manifest_path,\n                max_duration=self._max_duration,\n                min_duration=self._min_duration)\n            # sort (by duration) or batch-wise shuffle the manifest\n            if self._epoch == 0 and sortagrad:\n                manifest.sort(key=lambda x: x[\"duration\"])\n            else:\n                if shuffle_method == \"batch_shuffle\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=False)\n                elif shuffle_method == \"batch_shuffle_clipped\":\n                    manifest = self._batch_shuffle(\n                        manifest, batch_size, clipped=True)\n                elif shuffle_method == \"instance_shuffle\":\n                    self._rng.shuffle(manifest)\n                elif shuffle_method == None:\n                    pass\n                else:\n                    raise ValueError(\"Unknown shuffle method %s.\" %\n                                     shuffle_method)\n            # prepare batches\n            instance_reader = self._instance_reader_creator(manifest)\n            batch = []\n            for instance in instance_reader():\n                batch.append(instance)\n                if len(batch) == batch_size:\n                    yield self._padding_batch(batch, padding_to, flatten)\n                    batch = []\n            if len(batch) >= min_batch_size:\n                yield self._padding_batch(batch, padding_to, flatten)\n            self._epoch += 1\n\n        return batch_reader\n\n    @property\n    def feeding(self):\n        \"\"\"Returns data reader's feeding dict.\n\n        :return: Data feeding dict.\n        :rtype: dict\n        \"\"\"\n        return {\"audio_spectrogram\": 0, \"transcript_text\": 1}\n\n    @property\n    def vocab_size(self):\n        \"\"\"Return the vocabulary size.\n\n        :return: Vocabulary size.\n        :rtype: int\n        \"\"\"\n        return self._speech_featurizer.vocab_size\n\n    @property\n    def vocab_list(self):\n        \"\"\"Return the vocabulary in list.\n\n        :return: Vocabulary in list.\n        :rtype: list\n        \"\"\"\n        return self._speech_featurizer.vocab_list\n\n    def _parse_tar(self, file):\n        \"\"\"Parse a tar file to get a tarfile object\n        and a map containing tarinfoes\n        \"\"\"\n        result = {}\n        f = tarfile.open(file)\n        for tarinfo in f.getmembers():\n            result[tarinfo.name] = tarinfo\n        return f, result\n\n    def _get_file_object(self, file):\n        \"\"\"Get file object by file path.\n\n        If file startwith tar, it will return a tar file object\n        and cached tar file info for next reading request.\n        It will return file directly, if the type of file is not str.\n        \"\"\"\n        if file.startswith('tar:'):\n            tarpath, filename = file.split(':', 1)[1].split('#', 1)\n            if 'tar2info' not in self.local_data.__dict__:\n                self.local_data.tar2info = {}\n            if 'tar2object' not in self.local_data.__dict__:\n                self.local_data.tar2object = {}\n            if tarpath not in self.local_data.tar2info:\n                object, infoes = self._parse_tar(tarpath)\n                self.local_data.tar2info[tarpath] = infoes\n                self.local_data.tar2object[tarpath] = object\n            return self.local_data.tar2object[tarpath].extractfile(\n                self.local_data.tar2info[tarpath][filename])\n        else:\n            return open(file, 'r')\n\n    def _instance_reader_creator(self, manifest):\n        \"\"\"\n        Instance reader creator. Create a callable function to produce\n        instances of data.\n\n        Instance: a tuple of ndarray of audio spectrogram and a list of\n        token indices for transcript.\n        \"\"\"\n\n        def reader():\n            for instance in manifest:\n                yield instance\n\n        def mapper(instance):\n            return self.process_utterance(\n                self._get_file_object(instance[\"audio_filepath\"]),\n                instance[\"text\"])\n\n        return paddle.reader.xmap_readers(\n            mapper, reader, self._num_threads, 1024, order=True)\n\n    def _padding_batch(self, batch, padding_to=-1, flatten=False):\n        \"\"\"\n        Padding audio features with zeros to make them have the same shape (or\n        a user-defined shape) within one bach.\n\n        If ``padding_to`` is -1, the maximun shape in the batch will be used\n        as the target shape for padding. Otherwise, `padding_to` will be the\n        target shape (only refers to the second axis).\n\n        If `flatten` is True, features will be flatten to 1darray.\n        \"\"\"\n        new_batch = []\n        # get target shape\n        max_length = max([audio.shape[1] for audio, text in batch])\n        if padding_to != -1:\n            if padding_to < max_length:\n                raise ValueError(\"If padding_to is not -1, it should be larger \"\n                                 \"than any instance's shape in the batch\")\n            max_length = padding_to\n        # padding\n        for audio, text in batch:\n            padded_audio = np.zeros([audio.shape[0], max_length])\n            padded_audio[:, :audio.shape[1]] = audio\n            if flatten:\n                padded_audio = padded_audio.flatten()\n            new_batch.append((padded_audio, text))\n        return new_batch\n\n    def _batch_shuffle(self, manifest, batch_size, clipped=False):\n        \"\"\"Put similarly-sized instances into minibatches for better efficiency\n        and make a batch-wise shuffle.\n\n        1. Sort the audio clips by duration.\n        2. Generate a random number `k`, k in [0, batch_size).\n        3. Randomly shift `k` instances in order to create different batches\n           for different epochs. Create minibatches.\n        4. Shuffle the minibatches.\n\n        :param manifest: Manifest contents. List of dict.\n        :type manifest: list\n        :param batch_size: Batch size. This size is also used for generate\n                           a random number for batch shuffle.\n        :type batch_size: int\n        :param clipped: Whether to clip the heading (small shift) and trailing\n                        (incomplete batch) instances.\n        :type clipped: bool\n        :return: Batch shuffled mainifest.\n        :rtype: list\n        \"\"\"\n        manifest.sort(key=lambda x: x[\"duration\"])\n        shift_len = self._rng.randint(0, batch_size - 1)\n        batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)\n        self._rng.shuffle(batch_manifest)\n        batch_manifest = list(sum(batch_manifest, ()))\n        if not clipped:\n            res_len = len(manifest) - shift_len - len(batch_manifest)\n            batch_manifest.extend(manifest[-res_len:])\n            batch_manifest.extend(manifest[0:shift_len])\n        return batch_manifest\n", "path": "deep_speech_2/data_utils/data.py"}]}